Total coverage: 15683 (2%)of 1195164 




























  273 
  273 


  273 






























   29 








































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
// SPDX-License-Identifier: GPL-2.0
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/bitops.h>
#include <linux/cpumask.h>
#include <linux/export.h>
#include <linux/memblock.h>
#include <linux/numa.h>

/* These are not inline because of header tangles. */
#ifdef CONFIG_CPUMASK_OFFSTACK
/**
 * alloc_cpumask_var_node - allocate a struct cpumask on a given node
 * @mask: pointer to cpumask_var_t where the cpumask is returned
 * @flags: GFP_ flags
 * @node: memory node from which to allocate or %NUMA_NO_NODE
 *
 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
 * a nop returning a constant 1 (in <linux/cpumask.h>).
 *
 * Return: TRUE if memory allocation succeeded, FALSE otherwise.
 *
 * In addition, mask will be NULL if this fails.  Note that gcc is
 * usually smart enough to know that mask can never be NULL if
 * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case
 * too.
 */
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
{
        *mask = kmalloc_node(cpumask_size(), flags, node);

#ifdef CONFIG_DEBUG_PER_CPU_MAPS
        if (!*mask) {
                printk(KERN_ERR "=> alloc_cpumask_var: failed!\n");
                dump_stack();
        }
#endif

        return *mask != NULL;
}
EXPORT_SYMBOL(alloc_cpumask_var_node);

/**
 * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena.
 * @mask: pointer to cpumask_var_t where the cpumask is returned
 *
 * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
 * a nop (in <linux/cpumask.h>).
 * Either returns an allocated (zero-filled) cpumask, or causes the
 * system to panic.
 */
void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask)
{
        *mask = memblock_alloc_or_panic(cpumask_size(), SMP_CACHE_BYTES);
}

/**
 * free_cpumask_var - frees memory allocated for a struct cpumask.
 * @mask: cpumask to free
 *
 * This is safe on a NULL mask.
 */
void free_cpumask_var(cpumask_var_t mask)
{
        kfree(mask);
}
EXPORT_SYMBOL(free_cpumask_var);

/**
 * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var
 * @mask: cpumask to free
 */
void __init free_bootmem_cpumask_var(cpumask_var_t mask)
{
        memblock_free(mask, cpumask_size());
}
#endif

/**
 * cpumask_local_spread - select the i'th cpu based on NUMA distances
 * @i: index number
 * @node: local numa_node
 *
 * Return: online CPU according to a numa aware policy; local cpus are returned
 * first, followed by non-local ones, then it wraps around.
 *
 * For those who wants to enumerate all CPUs based on their NUMA distances,
 * i.e. call this function in a loop, like:
 *
 * for (i = 0; i < num_online_cpus(); i++) {
 *        cpu = cpumask_local_spread(i, node);
 *        do_something(cpu);
 * }
 *
 * There's a better alternative based on for_each()-like iterators:
 *
 *        for_each_numa_hop_mask(mask, node) {
 *                for_each_cpu_andnot(cpu, mask, prev)
 *                        do_something(cpu);
 *                prev = mask;
 *        }
 *
 * It's simpler and more verbose than above. Complexity of iterator-based
 * enumeration is O(sched_domains_numa_levels * nr_cpu_ids), while
 * cpumask_local_spread() when called for each cpu is
 * O(sched_domains_numa_levels * nr_cpu_ids * log(nr_cpu_ids)).
 */
unsigned int cpumask_local_spread(unsigned int i, int node)
{
        unsigned int cpu;

        /* Wrap: we always want a cpu. */
        i %= num_online_cpus();

        cpu = sched_numa_find_nth_cpu(cpu_online_mask, i, node);

        WARN_ON(cpu >= nr_cpu_ids);
        return cpu;
}
EXPORT_SYMBOL(cpumask_local_spread);

static DEFINE_PER_CPU(int, distribute_cpu_mask_prev);

/**
 * cpumask_any_and_distribute - Return an arbitrary cpu within src1p & src2p.
 * @src1p: first &cpumask for intersection
 * @src2p: second &cpumask for intersection
 *
 * Iterated calls using the same srcp1 and srcp2 will be distributed within
 * their intersection.
 *
 * Return: >= nr_cpu_ids if the intersection is empty.
 */
unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
                               const struct cpumask *src2p)
{
        unsigned int next, prev;

        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);

        next = cpumask_next_and_wrap(prev, src1p, src2p);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);

        return next;
}
EXPORT_SYMBOL(cpumask_any_and_distribute);

/**
 * cpumask_any_distribute - Return an arbitrary cpu from srcp
 * @srcp: &cpumask for selection
 *
 * Return: >= nr_cpu_ids if the intersection is empty.
 */
unsigned int cpumask_any_distribute(const struct cpumask *srcp)
{
        unsigned int next, prev;

        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);
        next = cpumask_next_wrap(prev, srcp);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);

        return next;
}
EXPORT_SYMBOL(cpumask_any_distribute);










   29 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM printk

#if !defined(_TRACE_PRINTK_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PRINTK_H

#include <linux/tracepoint.h>

TRACE_EVENT(console,
        TP_PROTO(const char *text, size_t len),

        TP_ARGS(text, len),

        TP_STRUCT__entry(
                __dynamic_array(char, msg, len + 1)
        ),

        TP_fast_assign(
                /*
                 * Each trace entry is printed in a new line.
                 * If the msg finishes with '\n', cut it off
                 * to avoid blank lines in the trace.
                 */
                if ((len > 0) && (text[len-1] == '\n'))
                        len -= 1;

                memcpy(__get_str(msg), text, len);
                __get_str(msg)[len] = 0;
        ),

        TP_printk("%s", __get_str(msg))
);
#endif /* _TRACE_PRINTK_H */

/* This part must be outside protection */
#include <trace/define_trace.h>



























































   22 




   22 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM csd

#if !defined(_TRACE_CSD_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_CSD_H

#include <linux/tracepoint.h>

TRACE_EVENT(csd_queue_cpu,

        TP_PROTO(const unsigned int cpu,
                unsigned long callsite,
                smp_call_func_t func,
                call_single_data_t *csd),

        TP_ARGS(cpu, callsite, func, csd),

        TP_STRUCT__entry(
                __field(unsigned int, cpu)
                __field(void *, callsite)
                __field(void *, func)
                __field(void *, csd)
                ),

            TP_fast_assign(
                __entry->cpu = cpu;
                __entry->callsite = (void *)callsite;
                __entry->func = func;
                __entry->csd  = csd;
                ),

        TP_printk("cpu=%u callsite=%pS func=%ps csd=%p",
                __entry->cpu, __entry->callsite, __entry->func, __entry->csd)
        );

/*
 * Tracepoints for a function which is called as an effect of smp_call_function.*
 */
DECLARE_EVENT_CLASS(csd_function,

        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),

        TP_ARGS(func, csd),

        TP_STRUCT__entry(
                __field(void *,        func)
                __field(void *,        csd)
        ),

        TP_fast_assign(
                __entry->func        = func;
                __entry->csd        = csd;
        ),

        TP_printk("func=%ps, csd=%p", __entry->func, __entry->csd)
);

DEFINE_EVENT(csd_function, csd_function_entry,
        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
        TP_ARGS(func, csd)
);

DEFINE_EVENT(csd_function, csd_function_exit,
        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
        TP_ARGS(func, csd)
);

#endif /* _TRACE_CSD_H */

/* This part must be outside protection */
#include <trace/define_trace.h>


























    2 



    2 





































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
// SPDX-License-Identifier: GPL-2.0-only

#include <linux/crc32.h>
#include <linux/linkage.h>
#include <linux/module.h>

#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/neon.h>
#include <asm/simd.h>

#include <crypto/internal/simd.h>

// The minimum input length to consider the 4-way interleaved code path
static const size_t min_len = 1024;

asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len);

asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len);
asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len);

u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
{
        if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
                return crc32_le_base(crc, p, len);

        if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
                kernel_neon_begin();
                crc = crc32_le_arm64_4way(crc, p, len);
                kernel_neon_end();

                p += round_down(len, 64);
                len %= 64;

                if (!len)
                        return crc;
        }

        return crc32_le_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32_le_arch);

u32 crc32c_arch(u32 crc, const u8 *p, size_t len)
{
        if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
                return crc32c_base(crc, p, len);

        if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
                kernel_neon_begin();
                crc = crc32c_le_arm64_4way(crc, p, len);
                kernel_neon_end();

                p += round_down(len, 64);
                len %= 64;

                if (!len)
                        return crc;
        }

        return crc32c_le_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32c_arch);

u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
{
        if (!alternative_has_cap_likely(ARM64_HAS_CRC32))
                return crc32_be_base(crc, p, len);

        if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) {
                kernel_neon_begin();
                crc = crc32_be_arm64_4way(crc, p, len);
                kernel_neon_end();

                p += round_down(len, 64);
                len %= 64;

                if (!len)
                        return crc;
        }

        return crc32_be_arm64(crc, p, len);
}
EXPORT_SYMBOL(crc32_be_arch);

u32 crc32_optimizations(void)
{
        if (alternative_has_cap_likely(ARM64_HAS_CRC32))
                return CRC32_LE_OPTIMIZATION |
                       CRC32_BE_OPTIMIZATION |
                       CRC32C_OPTIMIZATION;
        return 0;
}
EXPORT_SYMBOL(crc32_optimizations);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("arm64-optimized CRC32 functions");












 1809 





























































   20 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H

#include <linux/jump_label.h>
#include <linux/thread_info.h>

#define PREEMPT_NEED_RESCHED        BIT(32)
#define PREEMPT_ENABLED        (PREEMPT_NEED_RESCHED)

static inline int preempt_count(void)
{
        return READ_ONCE(current_thread_info()->preempt.count);
}

static inline void preempt_count_set(u64 pc)
{
        /* Preserve existing value of PREEMPT_NEED_RESCHED */
        WRITE_ONCE(current_thread_info()->preempt.count, pc);
}

#define init_task_preempt_count(p) do { \
        task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \
} while (0)

#define init_idle_preempt_count(p, cpu) do { \
        task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \
} while (0)

static inline void set_preempt_need_resched(void)
{
        current_thread_info()->preempt.need_resched = 0;
}

static inline void clear_preempt_need_resched(void)
{
        current_thread_info()->preempt.need_resched = 1;
}

static inline bool test_preempt_need_resched(void)
{
        return !current_thread_info()->preempt.need_resched;
}

static inline void __preempt_count_add(int val)
{
        u32 pc = READ_ONCE(current_thread_info()->preempt.count);
        pc += val;
        WRITE_ONCE(current_thread_info()->preempt.count, pc);
}

static inline void __preempt_count_sub(int val)
{
        u32 pc = READ_ONCE(current_thread_info()->preempt.count);
        pc -= val;
        WRITE_ONCE(current_thread_info()->preempt.count, pc);
}

static inline bool __preempt_count_dec_and_test(void)
{
        struct thread_info *ti = current_thread_info();
        u64 pc = READ_ONCE(ti->preempt_count);

        /* Update only the count field, leaving need_resched unchanged */
        WRITE_ONCE(ti->preempt.count, --pc);

        /*
         * If we wrote back all zeroes, then we're preemptible and in
         * need of a reschedule. Otherwise, we need to reload the
         * preempt_count in case the need_resched flag was cleared by an
         * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE
         * pair.
         */
        return !pc || !READ_ONCE(ti->preempt_count);
}

static inline bool should_resched(int preempt_offset)
{
        u64 pc = READ_ONCE(current_thread_info()->preempt_count);
        return pc == preempt_offset;
}

#ifdef CONFIG_PREEMPTION

void preempt_schedule(void);
void preempt_schedule_notrace(void);

#ifdef CONFIG_PREEMPT_DYNAMIC

DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
void dynamic_preempt_schedule(void);
#define __preempt_schedule()                dynamic_preempt_schedule()
void dynamic_preempt_schedule_notrace(void);
#define __preempt_schedule_notrace()        dynamic_preempt_schedule_notrace()

#else /* CONFIG_PREEMPT_DYNAMIC */

#define __preempt_schedule()                preempt_schedule()
#define __preempt_schedule_notrace()        preempt_schedule_notrace()

#endif /* CONFIG_PREEMPT_DYNAMIC */
#endif /* CONFIG_PREEMPTION */

#endif /* __ASM_PREEMPT_H */

























































































































































































































































































































































































































































































































































    6 



    6 
    6 




































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/act_mirred.c        packet mirroring and redirect actions
 *
 * Authors:        Jamal Hadi Salim (2002-4)
 *
 * TODO: Add ingress support (and socket redirect support)
 */

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/gfp.h>
#include <linux/if_arp.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/dst.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <linux/tc_act/tc_mirred.h>
#include <net/tc_act/tc_mirred.h>
#include <net/tc_wrapper.h>

static LIST_HEAD(mirred_list);
static DEFINE_SPINLOCK(mirred_list_lock);

#define MIRRED_NEST_LIMIT    4

#ifndef CONFIG_PREEMPT_RT
static u8 tcf_mirred_nest_level_inc_return(void)
{
        return __this_cpu_inc_return(softnet_data.xmit.sched_mirred_nest);
}

static void tcf_mirred_nest_level_dec(void)
{
        __this_cpu_dec(softnet_data.xmit.sched_mirred_nest);
}

#else
static u8 tcf_mirred_nest_level_inc_return(void)
{
        return current->net_xmit.sched_mirred_nest++;
}

static void tcf_mirred_nest_level_dec(void)
{
        current->net_xmit.sched_mirred_nest--;
}
#endif

static bool tcf_mirred_is_act_redirect(int action)
{
        return action == TCA_EGRESS_REDIR || action == TCA_INGRESS_REDIR;
}

static bool tcf_mirred_act_wants_ingress(int action)
{
        switch (action) {
        case TCA_EGRESS_REDIR:
        case TCA_EGRESS_MIRROR:
                return false;
        case TCA_INGRESS_REDIR:
        case TCA_INGRESS_MIRROR:
                return true;
        default:
                BUG();
        }
}

static bool tcf_mirred_can_reinsert(int action)
{
        switch (action) {
        case TC_ACT_SHOT:
        case TC_ACT_STOLEN:
        case TC_ACT_QUEUED:
        case TC_ACT_TRAP:
                return true;
        }
        return false;
}

static struct net_device *tcf_mirred_dev_dereference(struct tcf_mirred *m)
{
        return rcu_dereference_protected(m->tcfm_dev,
                                         lockdep_is_held(&m->tcf_lock));
}

static void tcf_mirred_release(struct tc_action *a)
{
        struct tcf_mirred *m = to_mirred(a);
        struct net_device *dev;

        spin_lock(&mirred_list_lock);
        list_del(&m->tcfm_list);
        spin_unlock(&mirred_list_lock);

        /* last reference to action, no need to lock */
        dev = rcu_dereference_protected(m->tcfm_dev, 1);
        netdev_put(dev, &m->tcfm_dev_tracker);
}

static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = {
        [TCA_MIRRED_PARMS]        = { .len = sizeof(struct tc_mirred) },
        [TCA_MIRRED_BLOCKID]        = NLA_POLICY_MIN(NLA_U32, 1),
};

static struct tc_action_ops act_mirred_ops;

static void tcf_mirred_replace_dev(struct tcf_mirred *m,
                                   struct net_device *ndev)
{
        struct net_device *odev;

        odev = rcu_replace_pointer(m->tcfm_dev, ndev,
                                   lockdep_is_held(&m->tcf_lock));
        netdev_put(odev, &m->tcfm_dev_tracker);
}

static int tcf_mirred_init(struct net *net, struct nlattr *nla,
                           struct nlattr *est, struct tc_action **a,
                           struct tcf_proto *tp,
                           u32 flags, struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_MIRRED_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        bool mac_header_xmit = false;
        struct tc_mirred *parm;
        struct tcf_mirred *m;
        bool exists = false;
        int ret, err;
        u32 index;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "Mirred requires attributes to be passed");
                return -EINVAL;
        }
        ret = nla_parse_nested_deprecated(tb, TCA_MIRRED_MAX, nla,
                                          mirred_policy, extack);
        if (ret < 0)
                return ret;
        if (!tb[TCA_MIRRED_PARMS]) {
                NL_SET_ERR_MSG_MOD(extack, "Missing required mirred parameters");
                return -EINVAL;
        }
        parm = nla_data(tb[TCA_MIRRED_PARMS]);
        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0)
                return err;
        exists = err;
        if (exists && bind)
                return ACT_P_BOUND;

        if (tb[TCA_MIRRED_BLOCKID] && parm->ifindex) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Cannot specify Block ID and dev simultaneously");
                if (exists)
                        tcf_idr_release(*a, bind);
                else
                        tcf_idr_cleanup(tn, index);

                return -EINVAL;
        }

        switch (parm->eaction) {
        case TCA_EGRESS_MIRROR:
        case TCA_EGRESS_REDIR:
        case TCA_INGRESS_REDIR:
        case TCA_INGRESS_MIRROR:
                break;
        default:
                if (exists)
                        tcf_idr_release(*a, bind);
                else
                        tcf_idr_cleanup(tn, index);
                NL_SET_ERR_MSG_MOD(extack, "Unknown mirred option");
                return -EINVAL;
        }

        if (!exists) {
                if (!parm->ifindex && !tb[TCA_MIRRED_BLOCKID]) {
                        tcf_idr_cleanup(tn, index);
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Must specify device or block");
                        return -EINVAL;
                }
                ret = tcf_idr_create_from_flags(tn, index, est, a,
                                                &act_mirred_ops, bind, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                return -EEXIST;
        }

        m = to_mirred(*a);
        if (ret == ACT_P_CREATED)
                INIT_LIST_HEAD(&m->tcfm_list);

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        spin_lock_bh(&m->tcf_lock);

        if (parm->ifindex) {
                struct net_device *ndev;

                ndev = dev_get_by_index(net, parm->ifindex);
                if (!ndev) {
                        spin_unlock_bh(&m->tcf_lock);
                        err = -ENODEV;
                        goto put_chain;
                }
                mac_header_xmit = dev_is_mac_header_xmit(ndev);
                tcf_mirred_replace_dev(m, ndev);
                netdev_tracker_alloc(ndev, &m->tcfm_dev_tracker, GFP_ATOMIC);
                m->tcfm_mac_header_xmit = mac_header_xmit;
                m->tcfm_blockid = 0;
        } else if (tb[TCA_MIRRED_BLOCKID]) {
                tcf_mirred_replace_dev(m, NULL);
                m->tcfm_mac_header_xmit = false;
                m->tcfm_blockid = nla_get_u32(tb[TCA_MIRRED_BLOCKID]);
        }
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        m->tcfm_eaction = parm->eaction;
        spin_unlock_bh(&m->tcf_lock);
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);

        if (ret == ACT_P_CREATED) {
                spin_lock(&mirred_list_lock);
                list_add(&m->tcfm_list, &mirred_list);
                spin_unlock(&mirred_list_lock);
        }

        return ret;
put_chain:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        tcf_idr_release(*a, bind);
        return err;
}

static int
tcf_mirred_forward(bool at_ingress, bool want_ingress, struct sk_buff *skb)
{
        int err;

        if (!want_ingress)
                err = tcf_dev_queue_xmit(skb, dev_queue_xmit);
        else if (!at_ingress)
                err = netif_rx(skb);
        else
                err = netif_receive_skb(skb);

        return err;
}

static int tcf_mirred_to_dev(struct sk_buff *skb, struct tcf_mirred *m,
                             struct net_device *dev,
                             const bool m_mac_header_xmit, int m_eaction,
                             int retval)
{
        struct sk_buff *skb_to_send = skb;
        bool want_ingress;
        bool is_redirect;
        bool expects_nh;
        bool at_ingress;
        bool dont_clone;
        int mac_len;
        bool at_nh;
        int err;

        is_redirect = tcf_mirred_is_act_redirect(m_eaction);
        if (unlikely(!(dev->flags & IFF_UP)) || !netif_carrier_ok(dev)) {
                net_notice_ratelimited("tc mirred to Houston: device %s is down\n",
                                       dev->name);
                goto err_cant_do;
        }

        /* we could easily avoid the clone only if called by ingress and clsact;
         * since we can't easily detect the clsact caller, skip clone only for
         * ingress - that covers the TC S/W datapath.
         */
        at_ingress = skb_at_tc_ingress(skb);
        dont_clone = skb_at_tc_ingress(skb) && is_redirect &&
                tcf_mirred_can_reinsert(retval);
        if (!dont_clone) {
                skb_to_send = skb_clone(skb, GFP_ATOMIC);
                if (!skb_to_send)
                        goto err_cant_do;
        }

        want_ingress = tcf_mirred_act_wants_ingress(m_eaction);

        /* All mirred/redirected skbs should clear previous ct info */
        nf_reset_ct(skb_to_send);
        if (want_ingress && !at_ingress) /* drop dst for egress -> ingress */
                skb_dst_drop(skb_to_send);

        expects_nh = want_ingress || !m_mac_header_xmit;
        at_nh = skb->data == skb_network_header(skb);
        if (at_nh != expects_nh) {
                mac_len = at_ingress ? skb->mac_len :
                          skb_network_offset(skb);
                if (expects_nh) {
                        /* target device/action expect data at nh */
                        skb_pull_rcsum(skb_to_send, mac_len);
                } else {
                        /* target device/action expect data at mac */
                        skb_push_rcsum(skb_to_send, mac_len);
                }
        }

        skb_to_send->skb_iif = skb->dev->ifindex;
        skb_to_send->dev = dev;

        if (is_redirect) {
                if (skb == skb_to_send)
                        retval = TC_ACT_CONSUMED;

                skb_set_redirected(skb_to_send, skb_to_send->tc_at_ingress);

                err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send);
        } else {
                err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send);
        }
        if (err)
                tcf_action_inc_overlimit_qstats(&m->common);

        return retval;

err_cant_do:
        if (is_redirect)
                retval = TC_ACT_SHOT;
        tcf_action_inc_overlimit_qstats(&m->common);
        return retval;
}

static int tcf_blockcast_redir(struct sk_buff *skb, struct tcf_mirred *m,
                               struct tcf_block *block, int m_eaction,
                               const u32 exception_ifindex, int retval)
{
        struct net_device *dev_prev = NULL;
        struct net_device *dev = NULL;
        unsigned long index;
        int mirred_eaction;

        mirred_eaction = tcf_mirred_act_wants_ingress(m_eaction) ?
                TCA_INGRESS_MIRROR : TCA_EGRESS_MIRROR;

        xa_for_each(&block->ports, index, dev) {
                if (index == exception_ifindex)
                        continue;

                if (!dev_prev)
                        goto assign_prev;

                tcf_mirred_to_dev(skb, m, dev_prev,
                                  dev_is_mac_header_xmit(dev),
                                  mirred_eaction, retval);
assign_prev:
                dev_prev = dev;
        }

        if (dev_prev)
                return tcf_mirred_to_dev(skb, m, dev_prev,
                                         dev_is_mac_header_xmit(dev_prev),
                                         m_eaction, retval);

        return retval;
}

static int tcf_blockcast_mirror(struct sk_buff *skb, struct tcf_mirred *m,
                                struct tcf_block *block, int m_eaction,
                                const u32 exception_ifindex, int retval)
{
        struct net_device *dev = NULL;
        unsigned long index;

        xa_for_each(&block->ports, index, dev) {
                if (index == exception_ifindex)
                        continue;

                tcf_mirred_to_dev(skb, m, dev,
                                  dev_is_mac_header_xmit(dev),
                                  m_eaction, retval);
        }

        return retval;
}

static int tcf_blockcast(struct sk_buff *skb, struct tcf_mirred *m,
                         const u32 blockid, struct tcf_result *res,
                         int retval)
{
        const u32 exception_ifindex = skb->dev->ifindex;
        struct tcf_block *block;
        bool is_redirect;
        int m_eaction;

        m_eaction = READ_ONCE(m->tcfm_eaction);
        is_redirect = tcf_mirred_is_act_redirect(m_eaction);

        /* we are already under rcu protection, so can call block lookup
         * directly.
         */
        block = tcf_block_lookup(dev_net(skb->dev), blockid);
        if (!block || xa_empty(&block->ports)) {
                tcf_action_inc_overlimit_qstats(&m->common);
                return retval;
        }

        if (is_redirect)
                return tcf_blockcast_redir(skb, m, block, m_eaction,
                                           exception_ifindex, retval);

        /* If it's not redirect, it is mirror */
        return tcf_blockcast_mirror(skb, m, block, m_eaction, exception_ifindex,
                                    retval);
}

TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb,
                                     const struct tc_action *a,
                                     struct tcf_result *res)
{
        struct tcf_mirred *m = to_mirred(a);
        int retval = READ_ONCE(m->tcf_action);
        unsigned int nest_level;
        bool m_mac_header_xmit;
        struct net_device *dev;
        int m_eaction;
        u32 blockid;

        nest_level = tcf_mirred_nest_level_inc_return();
        if (unlikely(nest_level > MIRRED_NEST_LIMIT)) {
                net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n",
                                     netdev_name(skb->dev));
                retval = TC_ACT_SHOT;
                goto dec_nest_level;
        }

        tcf_lastuse_update(&m->tcf_tm);
        tcf_action_update_bstats(&m->common, skb);

        blockid = READ_ONCE(m->tcfm_blockid);
        if (blockid) {
                retval = tcf_blockcast(skb, m, blockid, res, retval);
                goto dec_nest_level;
        }

        dev = rcu_dereference_bh(m->tcfm_dev);
        if (unlikely(!dev)) {
                pr_notice_once("tc mirred: target device is gone\n");
                tcf_action_inc_overlimit_qstats(&m->common);
                goto dec_nest_level;
        }

        m_mac_header_xmit = READ_ONCE(m->tcfm_mac_header_xmit);
        m_eaction = READ_ONCE(m->tcfm_eaction);

        retval = tcf_mirred_to_dev(skb, m, dev, m_mac_header_xmit, m_eaction,
                                   retval);

dec_nest_level:
        tcf_mirred_nest_level_dec();

        return retval;
}

static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
                             u64 drops, u64 lastuse, bool hw)
{
        struct tcf_mirred *m = to_mirred(a);
        struct tcf_t *tm = &m->tcf_tm;

        tcf_action_update_stats(a, bytes, packets, drops, hw);
        tm->lastuse = max_t(u64, tm->lastuse, lastuse);
}

static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind,
                           int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_mirred *m = to_mirred(a);
        struct tc_mirred opt = {
                .index   = m->tcf_index,
                .refcnt  = refcount_read(&m->tcf_refcnt) - ref,
                .bindcnt = atomic_read(&m->tcf_bindcnt) - bind,
        };
        struct net_device *dev;
        struct tcf_t t;
        u32 blockid;

        spin_lock_bh(&m->tcf_lock);
        opt.action = m->tcf_action;
        opt.eaction = m->tcfm_eaction;
        dev = tcf_mirred_dev_dereference(m);
        if (dev)
                opt.ifindex = dev->ifindex;

        if (nla_put(skb, TCA_MIRRED_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        blockid = m->tcfm_blockid;
        if (blockid && nla_put_u32(skb, TCA_MIRRED_BLOCKID, blockid))
                goto nla_put_failure;

        tcf_tm_dump(&t, &m->tcf_tm);
        if (nla_put_64bit(skb, TCA_MIRRED_TM, sizeof(t), &t, TCA_MIRRED_PAD))
                goto nla_put_failure;
        spin_unlock_bh(&m->tcf_lock);

        return skb->len;

nla_put_failure:
        spin_unlock_bh(&m->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static int mirred_device_event(struct notifier_block *unused,
                               unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct tcf_mirred *m;

        ASSERT_RTNL();
        if (event == NETDEV_UNREGISTER) {
                spin_lock(&mirred_list_lock);
                list_for_each_entry(m, &mirred_list, tcfm_list) {
                        spin_lock_bh(&m->tcf_lock);
                        if (tcf_mirred_dev_dereference(m) == dev) {
                                netdev_put(dev, &m->tcfm_dev_tracker);
                                /* Note : no rcu grace period necessary, as
                                 * net_device are already rcu protected.
                                 */
                                RCU_INIT_POINTER(m->tcfm_dev, NULL);
                        }
                        spin_unlock_bh(&m->tcf_lock);
                }
                spin_unlock(&mirred_list_lock);
        }

        return NOTIFY_DONE;
}

static struct notifier_block mirred_device_notifier = {
        .notifier_call = mirred_device_event,
};

static void tcf_mirred_dev_put(void *priv)
{
        struct net_device *dev = priv;

        dev_put(dev);
}

static struct net_device *
tcf_mirred_get_dev(const struct tc_action *a,
                   tc_action_priv_destructor *destructor)
{
        struct tcf_mirred *m = to_mirred(a);
        struct net_device *dev;

        rcu_read_lock();
        dev = rcu_dereference(m->tcfm_dev);
        if (dev) {
                dev_hold(dev);
                *destructor = tcf_mirred_dev_put;
        }
        rcu_read_unlock();

        return dev;
}

static size_t tcf_mirred_get_fill_size(const struct tc_action *act)
{
        return nla_total_size(sizeof(struct tc_mirred));
}

static void tcf_offload_mirred_get_dev(struct flow_action_entry *entry,
                                       const struct tc_action *act)
{
        entry->dev = act->ops->get_dev(act, &entry->destructor);
        if (!entry->dev)
                return;
        entry->destructor_priv = entry->dev;
}

static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data,
                                        u32 *index_inc, bool bind,
                                        struct netlink_ext_ack *extack)
{
        if (bind) {
                struct flow_action_entry *entry = entry_data;

                if (is_tcf_mirred_egress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_egress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_redirect(act)) {
                        entry->id = FLOW_ACTION_REDIRECT_INGRESS;
                        tcf_offload_mirred_get_dev(entry, act);
                } else if (is_tcf_mirred_ingress_mirror(act)) {
                        entry->id = FLOW_ACTION_MIRRED_INGRESS;
                        tcf_offload_mirred_get_dev(entry, act);
                } else {
                        NL_SET_ERR_MSG_MOD(extack, "Unsupported mirred offload");
                        return -EOPNOTSUPP;
                }
                *index_inc = 1;
        } else {
                struct flow_offload_action *fl_action = entry_data;

                if (is_tcf_mirred_egress_redirect(act))
                        fl_action->id = FLOW_ACTION_REDIRECT;
                else if (is_tcf_mirred_egress_mirror(act))
                        fl_action->id = FLOW_ACTION_MIRRED;
                else if (is_tcf_mirred_ingress_redirect(act))
                        fl_action->id = FLOW_ACTION_REDIRECT_INGRESS;
                else if (is_tcf_mirred_ingress_mirror(act))
                        fl_action->id = FLOW_ACTION_MIRRED_INGRESS;
                else
                        return -EOPNOTSUPP;
        }

        return 0;
}

static struct tc_action_ops act_mirred_ops = {
        .kind                =        "mirred",
        .id                =        TCA_ID_MIRRED,
        .owner                =        THIS_MODULE,
        .act                =        tcf_mirred_act,
        .stats_update        =        tcf_stats_update,
        .dump                =        tcf_mirred_dump,
        .cleanup        =        tcf_mirred_release,
        .init                =        tcf_mirred_init,
        .get_fill_size        =        tcf_mirred_get_fill_size,
        .offload_act_setup =        tcf_mirred_offload_act_setup,
        .size                =        sizeof(struct tcf_mirred),
        .get_dev        =        tcf_mirred_get_dev,
};
MODULE_ALIAS_NET_ACT("mirred");

static __net_init int mirred_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);

        return tc_action_net_init(net, tn, &act_mirred_ops);
}

static void __net_exit mirred_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_mirred_ops.net_id);
}

static struct pernet_operations mirred_net_ops = {
        .init = mirred_init_net,
        .exit_batch = mirred_exit_net,
        .id   = &act_mirred_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

MODULE_AUTHOR("Jamal Hadi Salim(2002)");
MODULE_DESCRIPTION("Device Mirror/redirect actions");
MODULE_LICENSE("GPL");

static int __init mirred_init_module(void)
{
        int err = register_netdevice_notifier(&mirred_device_notifier);
        if (err)
                return err;

        pr_info("Mirror/redirect action on\n");
        err = tcf_register_action(&act_mirred_ops, &mirred_net_ops);
        if (err)
                unregister_netdevice_notifier(&mirred_device_notifier);

        return err;
}

static void __exit mirred_cleanup_module(void)
{
        tcf_unregister_action(&act_mirred_ops, &mirred_net_ops);
        unregister_netdevice_notifier(&mirred_device_notifier);
}

module_init(mirred_init_module);
module_exit(mirred_cleanup_module);
























































































































   29 
   29 





































































































































































   29 













   29 
   29 













   29 








   29 

   29 

    4 

   29 










   29 

   29 










   29 

















   29 






















































































































































































   29 


















































































































































































































































































































































































































































































































































































































































































































































































   28 
























































































   28 


   28 



   28 






























   28 




























    6 


   28 








   28 























   28 







    6 




















   28 




















































































































































































































































































































































































   28 












   28 

























   28 
























   28 

































   29 





   29 












    4 




    4 















    4 








































































































   29 











   29 


























   29 
   29 
























   29 






   29 







   29 

   29 




   29 








   29 














   29 










   29 
















   29 

   29 


   29 


   29 


   29 



























   29 










   29 







   29 






   29 



   29 








































   29 












   29 







   29 

    3 



   29 



   29 



   29 

    3 














    4 
   29 
   29 












   29 





   29 











































































































































































































































































































































































































   29 

    4 

   29 












   29 

   29 
































































































   29 






















   29 










   28 







   28 
























   28 
   28 



   28 
   28 



























   29 








   29 










   28 



























   28 














   28 









































   29 















   29 
   29 








   29 


   29 








   29 











   28 
   29 

   29 




   28 





   29 
   29 

































   29 
















   29 















   29 




   29 

























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 















    4 

   29 


   29 

















































    3 






























































































































































































































































































































































   29 



















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/printk.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 * Modified to make sys_syslog() more flexible: added commands to
 * return the last 4k of kernel messages, regardless of whether
 * they've been read or not.  Added option to suppress kernel printk's
 * to the console.  Added hook for sending the console messages
 * elsewhere, in preparation for a serial line console (someday).
 * Ted Ts'o, 2/11/93.
 * Modified for sysctl support, 1/8/97, Chris Horn.
 * Fixed SMP synchronization, 08/08/99, Manfred Spraul
 *     manfred@colorfullife.com
 * Rewrote bits to get rid of console_lock
 *        01Mar01 Andrew Morton
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/tty.h>
#include <linux/tty_driver.h>
#include <linux/console.h>
#include <linux/init.h>
#include <linux/jiffies.h>
#include <linux/nmi.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/security.h>
#include <linux/memblock.h>
#include <linux/syscalls.h>
#include <linux/syscore_ops.h>
#include <linux/vmcore_info.h>
#include <linux/ratelimit.h>
#include <linux/kmsg_dump.h>
#include <linux/syslog.h>
#include <linux/cpu.h>
#include <linux/rculist.h>
#include <linux/poll.h>
#include <linux/irq_work.h>
#include <linux/ctype.h>
#include <linux/uio.h>
#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
#include <linux/sched/task_stack.h>

#include <linux/uaccess.h>
#include <asm/sections.h>

#include <trace/events/initcall.h>
#define CREATE_TRACE_POINTS
#include <trace/events/printk.h>

#include "printk_ringbuffer.h"
#include "console_cmdline.h"
#include "braille.h"
#include "internal.h"

int console_printk[4] = {
        CONSOLE_LOGLEVEL_DEFAULT,        /* console_loglevel */
        MESSAGE_LOGLEVEL_DEFAULT,        /* default_message_loglevel */
        CONSOLE_LOGLEVEL_MIN,                /* minimum_console_loglevel */
        CONSOLE_LOGLEVEL_DEFAULT,        /* default_console_loglevel */
};
EXPORT_SYMBOL_GPL(console_printk);

atomic_t ignore_console_lock_warning __read_mostly = ATOMIC_INIT(0);
EXPORT_SYMBOL(ignore_console_lock_warning);

EXPORT_TRACEPOINT_SYMBOL_GPL(console);

/*
 * Low level drivers may need that to know if they can schedule in
 * their unblank() callback or not. So let's export it.
 */
int oops_in_progress;
EXPORT_SYMBOL(oops_in_progress);

/*
 * console_mutex protects console_list updates and console->flags updates.
 * The flags are synchronized only for consoles that are registered, i.e.
 * accessible via the console list.
 */
static DEFINE_MUTEX(console_mutex);

/*
 * console_sem protects updates to console->seq
 * and also provides serialization for console printing.
 */
static DEFINE_SEMAPHORE(console_sem, 1);
HLIST_HEAD(console_list);
EXPORT_SYMBOL_GPL(console_list);
DEFINE_STATIC_SRCU(console_srcu);

/*
 * System may need to suppress printk message under certain
 * circumstances, like after kernel panic happens.
 */
int __read_mostly suppress_printk;

#ifdef CONFIG_LOCKDEP
static struct lockdep_map console_lock_dep_map = {
        .name = "console_lock"
};

void lockdep_assert_console_list_lock_held(void)
{
        lockdep_assert_held(&console_mutex);
}
EXPORT_SYMBOL(lockdep_assert_console_list_lock_held);
#endif

#ifdef CONFIG_DEBUG_LOCK_ALLOC
bool console_srcu_read_lock_is_held(void)
{
        return srcu_read_lock_held(&console_srcu);
}
EXPORT_SYMBOL(console_srcu_read_lock_is_held);
#endif

enum devkmsg_log_bits {
        __DEVKMSG_LOG_BIT_ON = 0,
        __DEVKMSG_LOG_BIT_OFF,
        __DEVKMSG_LOG_BIT_LOCK,
};

enum devkmsg_log_masks {
        DEVKMSG_LOG_MASK_ON             = BIT(__DEVKMSG_LOG_BIT_ON),
        DEVKMSG_LOG_MASK_OFF            = BIT(__DEVKMSG_LOG_BIT_OFF),
        DEVKMSG_LOG_MASK_LOCK           = BIT(__DEVKMSG_LOG_BIT_LOCK),
};

/* Keep both the 'on' and 'off' bits clear, i.e. ratelimit by default: */
#define DEVKMSG_LOG_MASK_DEFAULT        0

static unsigned int __read_mostly devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;

static int __control_devkmsg(char *str)
{
        size_t len;

        if (!str)
                return -EINVAL;

        len = str_has_prefix(str, "on");
        if (len) {
                devkmsg_log = DEVKMSG_LOG_MASK_ON;
                return len;
        }

        len = str_has_prefix(str, "off");
        if (len) {
                devkmsg_log = DEVKMSG_LOG_MASK_OFF;
                return len;
        }

        len = str_has_prefix(str, "ratelimit");
        if (len) {
                devkmsg_log = DEVKMSG_LOG_MASK_DEFAULT;
                return len;
        }

        return -EINVAL;
}

static int __init control_devkmsg(char *str)
{
        if (__control_devkmsg(str) < 0) {
                pr_warn("printk.devkmsg: bad option string '%s'\n", str);
                return 1;
        }

        /*
         * Set sysctl string accordingly:
         */
        if (devkmsg_log == DEVKMSG_LOG_MASK_ON)
                strscpy(devkmsg_log_str, "on");
        else if (devkmsg_log == DEVKMSG_LOG_MASK_OFF)
                strscpy(devkmsg_log_str, "off");
        /* else "ratelimit" which is set by default. */

        /*
         * Sysctl cannot change it anymore. The kernel command line setting of
         * this parameter is to force the setting to be permanent throughout the
         * runtime of the system. This is a precation measure against userspace
         * trying to be a smarta** and attempting to change it up on us.
         */
        devkmsg_log |= DEVKMSG_LOG_MASK_LOCK;

        return 1;
}
__setup("printk.devkmsg=", control_devkmsg);

char devkmsg_log_str[DEVKMSG_STR_MAX_SIZE] = "ratelimit";
#if defined(CONFIG_PRINTK) && defined(CONFIG_SYSCTL)
int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write,
                              void *buffer, size_t *lenp, loff_t *ppos)
{
        char old_str[DEVKMSG_STR_MAX_SIZE];
        unsigned int old;
        int err;

        if (write) {
                if (devkmsg_log & DEVKMSG_LOG_MASK_LOCK)
                        return -EINVAL;

                old = devkmsg_log;
                strscpy(old_str, devkmsg_log_str);
        }

        err = proc_dostring(table, write, buffer, lenp, ppos);
        if (err)
                return err;

        if (write) {
                err = __control_devkmsg(devkmsg_log_str);

                /*
                 * Do not accept an unknown string OR a known string with
                 * trailing crap...
                 */
                if (err < 0 || (err + 1 != *lenp)) {

                        /* ... and restore old setting. */
                        devkmsg_log = old;
                        strscpy(devkmsg_log_str, old_str);

                        return -EINVAL;
                }
        }

        return 0;
}
#endif /* CONFIG_PRINTK && CONFIG_SYSCTL */

/**
 * console_list_lock - Lock the console list
 *
 * For console list or console->flags updates
 */
void console_list_lock(void)
{
        /*
         * In unregister_console() and console_force_preferred_locked(),
         * synchronize_srcu() is called with the console_list_lock held.
         * Therefore it is not allowed that the console_list_lock is taken
         * with the srcu_lock held.
         *
         * Detecting if this context is really in the read-side critical
         * section is only possible if the appropriate debug options are
         * enabled.
         */
        WARN_ON_ONCE(debug_lockdep_rcu_enabled() &&
                     srcu_read_lock_held(&console_srcu));

        mutex_lock(&console_mutex);
}
EXPORT_SYMBOL(console_list_lock);

/**
 * console_list_unlock - Unlock the console list
 *
 * Counterpart to console_list_lock()
 */
void console_list_unlock(void)
{
        mutex_unlock(&console_mutex);
}
EXPORT_SYMBOL(console_list_unlock);

/**
 * console_srcu_read_lock - Register a new reader for the
 *        SRCU-protected console list
 *
 * Use for_each_console_srcu() to iterate the console list
 *
 * Context: Any context.
 * Return: A cookie to pass to console_srcu_read_unlock().
 */
int console_srcu_read_lock(void)
        __acquires(&console_srcu)
{
        return srcu_read_lock_nmisafe(&console_srcu);
}
EXPORT_SYMBOL(console_srcu_read_lock);

/**
 * console_srcu_read_unlock - Unregister an old reader from
 *        the SRCU-protected console list
 * @cookie: cookie returned from console_srcu_read_lock()
 *
 * Counterpart to console_srcu_read_lock()
 */
void console_srcu_read_unlock(int cookie)
        __releases(&console_srcu)
{
        srcu_read_unlock_nmisafe(&console_srcu, cookie);
}
EXPORT_SYMBOL(console_srcu_read_unlock);

/*
 * Helper macros to handle lockdep when locking/unlocking console_sem. We use
 * macros instead of functions so that _RET_IP_ contains useful information.
 */
#define down_console_sem() do { \
        down(&console_sem);\
        mutex_acquire(&console_lock_dep_map, 0, 0, _RET_IP_);\
} while (0)

static int __down_trylock_console_sem(unsigned long ip)
{
        int lock_failed;
        unsigned long flags;

        /*
         * Here and in __up_console_sem() we need to be in safe mode,
         * because spindump/WARN/etc from under console ->lock will
         * deadlock in printk()->down_trylock_console_sem() otherwise.
         */
        printk_safe_enter_irqsave(flags);
        lock_failed = down_trylock(&console_sem);
        printk_safe_exit_irqrestore(flags);

        if (lock_failed)
                return 1;
        mutex_acquire(&console_lock_dep_map, 0, 1, ip);
        return 0;
}
#define down_trylock_console_sem() __down_trylock_console_sem(_RET_IP_)

static void __up_console_sem(unsigned long ip)
{
        unsigned long flags;

        mutex_release(&console_lock_dep_map, ip);

        printk_safe_enter_irqsave(flags);
        up(&console_sem);
        printk_safe_exit_irqrestore(flags);
}
#define up_console_sem() __up_console_sem(_RET_IP_)

static bool panic_in_progress(void)
{
        return unlikely(atomic_read(&panic_cpu) != PANIC_CPU_INVALID);
}

/* Return true if a panic is in progress on the current CPU. */
bool this_cpu_in_panic(void)
{
        /*
         * We can use raw_smp_processor_id() here because it is impossible for
         * the task to be migrated to the panic_cpu, or away from it. If
         * panic_cpu has already been set, and we're not currently executing on
         * that CPU, then we never will be.
         */
        return unlikely(atomic_read(&panic_cpu) == raw_smp_processor_id());
}

/*
 * Return true if a panic is in progress on a remote CPU.
 *
 * On true, the local CPU should immediately release any printing resources
 * that may be needed by the panic CPU.
 */
bool other_cpu_in_panic(void)
{
        return (panic_in_progress() && !this_cpu_in_panic());
}

/*
 * This is used for debugging the mess that is the VT code by
 * keeping track if we have the console semaphore held. It's
 * definitely not the perfect debug tool (we don't know if _WE_
 * hold it and are racing, but it helps tracking those weird code
 * paths in the console code where we end up in places I want
 * locked without the console semaphore held).
 */
static int console_locked;

/*
 *        Array of consoles built from command line options (console=)
 */

#define MAX_CMDLINECONSOLES 8

static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];

static int preferred_console = -1;
int console_set_on_cmdline;
EXPORT_SYMBOL(console_set_on_cmdline);

/* Flag: console code may call schedule() */
static int console_may_schedule;

enum con_msg_format_flags {
        MSG_FORMAT_DEFAULT        = 0,
        MSG_FORMAT_SYSLOG        = (1 << 0),
};

static int console_msg_format = MSG_FORMAT_DEFAULT;

/*
 * The printk log buffer consists of a sequenced collection of records, each
 * containing variable length message text. Every record also contains its
 * own meta-data (@info).
 *
 * Every record meta-data carries the timestamp in microseconds, as well as
 * the standard userspace syslog level and syslog facility. The usual kernel
 * messages use LOG_KERN; userspace-injected messages always carry a matching
 * syslog facility, by default LOG_USER. The origin of every message can be
 * reliably determined that way.
 *
 * The human readable log message of a record is available in @text, the
 * length of the message text in @text_len. The stored message is not
 * terminated.
 *
 * Optionally, a record can carry a dictionary of properties (key/value
 * pairs), to provide userspace with a machine-readable message context.
 *
 * Examples for well-defined, commonly used property names are:
 *   DEVICE=b12:8               device identifier
 *                                b12:8         block dev_t
 *                                c127:3        char dev_t
 *                                n8            netdev ifindex
 *                                +sound:card0  subsystem:devname
 *   SUBSYSTEM=pci              driver-core subsystem name
 *
 * Valid characters in property names are [a-zA-Z0-9.-_]. Property names
 * and values are terminated by a '\0' character.
 *
 * Example of record values:
 *   record.text_buf                = "it's a line" (unterminated)
 *   record.info.seq                = 56
 *   record.info.ts_nsec            = 36863
 *   record.info.text_len           = 11
 *   record.info.facility           = 0 (LOG_KERN)
 *   record.info.flags              = 0
 *   record.info.level              = 3 (LOG_ERR)
 *   record.info.caller_id          = 299 (task 299)
 *   record.info.dev_info.subsystem = "pci" (terminated)
 *   record.info.dev_info.device    = "+pci:0000:00:01.0" (terminated)
 *
 * The 'struct printk_info' buffer must never be directly exported to
 * userspace, it is a kernel-private implementation detail that might
 * need to be changed in the future, when the requirements change.
 *
 * /dev/kmsg exports the structured data in the following line format:
 *   "<level>,<sequnum>,<timestamp>,<contflag>[,additional_values, ... ];<message text>\n"
 *
 * Users of the export format should ignore possible additional values
 * separated by ',', and find the message after the ';' character.
 *
 * The optional key/value pairs are attached as continuation lines starting
 * with a space character and terminated by a newline. All possible
 * non-prinatable characters are escaped in the "\xff" notation.
 */

/* syslog_lock protects syslog_* variables and write access to clear_seq. */
static DEFINE_MUTEX(syslog_lock);

/*
 * Specifies if a legacy console is registered. If legacy consoles are
 * present, it is necessary to perform the console lock/unlock dance
 * whenever console flushing should occur.
 */
bool have_legacy_console;

/*
 * Specifies if an nbcon console is registered. If nbcon consoles are present,
 * synchronous printing of legacy consoles will not occur during panic until
 * the backtrace has been stored to the ringbuffer.
 */
bool have_nbcon_console;

/*
 * Specifies if a boot console is registered. If boot consoles are present,
 * nbcon consoles cannot print simultaneously and must be synchronized by
 * the console lock. This is because boot consoles and nbcon consoles may
 * have mapped the same hardware.
 */
bool have_boot_console;

/* See printk_legacy_allow_panic_sync() for details. */
bool legacy_allow_panic_sync;

#ifdef CONFIG_PRINTK
DECLARE_WAIT_QUEUE_HEAD(log_wait);
static DECLARE_WAIT_QUEUE_HEAD(legacy_wait);
/* All 3 protected by @syslog_lock. */
/* the next printk record to read by syslog(READ) or /proc/kmsg */
static u64 syslog_seq;
static size_t syslog_partial;
static bool syslog_time;

/* True when _all_ printer threads are available for printing. */
bool printk_kthreads_running;

struct latched_seq {
        seqcount_latch_t        latch;
        u64                        val[2];
};

/*
 * The next printk record to read after the last 'clear' command. There are
 * two copies (updated with seqcount_latch) so that reads can locklessly
 * access a valid value. Writers are synchronized by @syslog_lock.
 */
static struct latched_seq clear_seq = {
        .latch                = SEQCNT_LATCH_ZERO(clear_seq.latch),
        .val[0]                = 0,
        .val[1]                = 0,
};

#define LOG_LEVEL(v)                ((v) & 0x07)
#define LOG_FACILITY(v)                ((v) >> 3 & 0xff)

/* record buffer */
#define LOG_ALIGN __alignof__(unsigned long)
#define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT)
#define LOG_BUF_LEN_MAX ((u32)1 << 31)
static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN);
static char *log_buf = __log_buf;
static u32 log_buf_len = __LOG_BUF_LEN;

/*
 * Define the average message size. This only affects the number of
 * descriptors that will be available. Underestimating is better than
 * overestimating (too many available descriptors is better than not enough).
 */
#define PRB_AVGBITS 5        /* 32 character average length */

#if CONFIG_LOG_BUF_SHIFT <= PRB_AVGBITS
#error CONFIG_LOG_BUF_SHIFT value too small.
#endif
_DEFINE_PRINTKRB(printk_rb_static, CONFIG_LOG_BUF_SHIFT - PRB_AVGBITS,
                 PRB_AVGBITS, &__log_buf[0]);

static struct printk_ringbuffer printk_rb_dynamic;

struct printk_ringbuffer *prb = &printk_rb_static;

/*
 * We cannot access per-CPU data (e.g. per-CPU flush irq_work) before
 * per_cpu_areas are initialised. This variable is set to true when
 * it's safe to access per-CPU data.
 */
static bool __printk_percpu_data_ready __ro_after_init;

bool printk_percpu_data_ready(void)
{
        return __printk_percpu_data_ready;
}

/* Must be called under syslog_lock. */
static void latched_seq_write(struct latched_seq *ls, u64 val)
{
        write_seqcount_latch_begin(&ls->latch);
        ls->val[0] = val;
        write_seqcount_latch(&ls->latch);
        ls->val[1] = val;
        write_seqcount_latch_end(&ls->latch);
}

/* Can be called from any context. */
static u64 latched_seq_read_nolock(struct latched_seq *ls)
{
        unsigned int seq;
        unsigned int idx;
        u64 val;

        do {
                seq = read_seqcount_latch(&ls->latch);
                idx = seq & 0x1;
                val = ls->val[idx];
        } while (read_seqcount_latch_retry(&ls->latch, seq));

        return val;
}

/* Return log buffer address */
char *log_buf_addr_get(void)
{
        return log_buf;
}

/* Return log buffer size */
u32 log_buf_len_get(void)
{
        return log_buf_len;
}

/*
 * Define how much of the log buffer we could take at maximum. The value
 * must be greater than two. Note that only half of the buffer is available
 * when the index points to the middle.
 */
#define MAX_LOG_TAKE_PART 4
static const char trunc_msg[] = "<truncated>";

static void truncate_msg(u16 *text_len, u16 *trunc_msg_len)
{
        /*
         * The message should not take the whole buffer. Otherwise, it might
         * get removed too soon.
         */
        u32 max_text_len = log_buf_len / MAX_LOG_TAKE_PART;

        if (*text_len > max_text_len)
                *text_len = max_text_len;

        /* enable the warning message (if there is room) */
        *trunc_msg_len = strlen(trunc_msg);
        if (*text_len >= *trunc_msg_len)
                *text_len -= *trunc_msg_len;
        else
                *trunc_msg_len = 0;
}

int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT);

static int syslog_action_restricted(int type)
{
        if (dmesg_restrict)
                return 1;
        /*
         * Unless restricted, we allow "read all" and "get buffer size"
         * for everybody.
         */
        return type != SYSLOG_ACTION_READ_ALL &&
               type != SYSLOG_ACTION_SIZE_BUFFER;
}

static int check_syslog_permissions(int type, int source)
{
        /*
         * If this is from /proc/kmsg and we've already opened it, then we've
         * already done the capabilities checks at open time.
         */
        if (source == SYSLOG_FROM_PROC && type != SYSLOG_ACTION_OPEN)
                goto ok;

        if (syslog_action_restricted(type)) {
                if (capable(CAP_SYSLOG))
                        goto ok;
                return -EPERM;
        }
ok:
        return security_syslog(type);
}

static void append_char(char **pp, char *e, char c)
{
        if (*pp < e)
                *(*pp)++ = c;
}

static ssize_t info_print_ext_header(char *buf, size_t size,
                                     struct printk_info *info)
{
        u64 ts_usec = info->ts_nsec;
        char caller[20];
#ifdef CONFIG_PRINTK_CALLER
        u32 id = info->caller_id;

        snprintf(caller, sizeof(caller), ",caller=%c%u",
                 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
#else
        caller[0] = '\0';
#endif

        do_div(ts_usec, 1000);

        return scnprintf(buf, size, "%u,%llu,%llu,%c%s;",
                         (info->facility << 3) | info->level, info->seq,
                         ts_usec, info->flags & LOG_CONT ? 'c' : '-', caller);
}

static ssize_t msg_add_ext_text(char *buf, size_t size,
                                const char *text, size_t text_len,
                                unsigned char endc)
{
        char *p = buf, *e = buf + size;
        size_t i;

        /* escape non-printable characters */
        for (i = 0; i < text_len; i++) {
                unsigned char c = text[i];

                if (c < ' ' || c >= 127 || c == '\\')
                        p += scnprintf(p, e - p, "\\x%02x", c);
                else
                        append_char(&p, e, c);
        }
        append_char(&p, e, endc);

        return p - buf;
}

static ssize_t msg_add_dict_text(char *buf, size_t size,
                                 const char *key, const char *val)
{
        size_t val_len = strlen(val);
        ssize_t len;

        if (!val_len)
                return 0;

        len = msg_add_ext_text(buf, size, "", 0, ' ');        /* dict prefix */
        len += msg_add_ext_text(buf + len, size - len, key, strlen(key), '=');
        len += msg_add_ext_text(buf + len, size - len, val, val_len, '\n');

        return len;
}

static ssize_t msg_print_ext_body(char *buf, size_t size,
                                  char *text, size_t text_len,
                                  struct dev_printk_info *dev_info)
{
        ssize_t len;

        len = msg_add_ext_text(buf, size, text, text_len, '\n');

        if (!dev_info)
                goto out;

        len += msg_add_dict_text(buf + len, size - len, "SUBSYSTEM",
                                 dev_info->subsystem);
        len += msg_add_dict_text(buf + len, size - len, "DEVICE",
                                 dev_info->device);
out:
        return len;
}

/* /dev/kmsg - userspace message inject/listen interface */
struct devkmsg_user {
        atomic64_t seq;
        struct ratelimit_state rs;
        struct mutex lock;
        struct printk_buffers pbufs;
};

static __printf(3, 4) __cold
int devkmsg_emit(int facility, int level, const char *fmt, ...)
{
        va_list args;
        int r;

        va_start(args, fmt);
        r = vprintk_emit(facility, level, NULL, fmt, args);
        va_end(args);

        return r;
}

static ssize_t devkmsg_write(struct kiocb *iocb, struct iov_iter *from)
{
        char *buf, *line;
        int level = default_message_loglevel;
        int facility = 1;        /* LOG_USER */
        struct file *file = iocb->ki_filp;
        struct devkmsg_user *user = file->private_data;
        size_t len = iov_iter_count(from);
        ssize_t ret = len;

        if (len > PRINTKRB_RECORD_MAX)
                return -EINVAL;

        /* Ignore when user logging is disabled. */
        if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
                return len;

        /* Ratelimit when not explicitly enabled. */
        if (!(devkmsg_log & DEVKMSG_LOG_MASK_ON)) {
                if (!___ratelimit(&user->rs, current->comm))
                        return ret;
        }

        buf = kmalloc(len+1, GFP_KERNEL);
        if (buf == NULL)
                return -ENOMEM;

        buf[len] = '\0';
        if (!copy_from_iter_full(buf, len, from)) {
                kfree(buf);
                return -EFAULT;
        }

        /*
         * Extract and skip the syslog prefix <[0-9]*>. Coming from userspace
         * the decimal value represents 32bit, the lower 3 bit are the log
         * level, the rest are the log facility.
         *
         * If no prefix or no userspace facility is specified, we
         * enforce LOG_USER, to be able to reliably distinguish
         * kernel-generated messages from userspace-injected ones.
         */
        line = buf;
        if (line[0] == '<') {
                char *endp = NULL;
                unsigned int u;

                u = simple_strtoul(line + 1, &endp, 10);
                if (endp && endp[0] == '>') {
                        level = LOG_LEVEL(u);
                        if (LOG_FACILITY(u) != 0)
                                facility = LOG_FACILITY(u);
                        endp++;
                        line = endp;
                }
        }

        devkmsg_emit(facility, level, "%s", line);
        kfree(buf);
        return ret;
}

static ssize_t devkmsg_read(struct file *file, char __user *buf,
                            size_t count, loff_t *ppos)
{
        struct devkmsg_user *user = file->private_data;
        char *outbuf = &user->pbufs.outbuf[0];
        struct printk_message pmsg = {
                .pbufs = &user->pbufs,
        };
        ssize_t ret;

        ret = mutex_lock_interruptible(&user->lock);
        if (ret)
                return ret;

        if (!printk_get_next_message(&pmsg, atomic64_read(&user->seq), true, false)) {
                if (file->f_flags & O_NONBLOCK) {
                        ret = -EAGAIN;
                        goto out;
                }

                /*
                 * Guarantee this task is visible on the waitqueue before
                 * checking the wake condition.
                 *
                 * The full memory barrier within set_current_state() of
                 * prepare_to_wait_event() pairs with the full memory barrier
                 * within wq_has_sleeper().
                 *
                 * This pairs with __wake_up_klogd:A.
                 */
                ret = wait_event_interruptible(log_wait,
                                printk_get_next_message(&pmsg, atomic64_read(&user->seq), true,
                                                        false)); /* LMM(devkmsg_read:A) */
                if (ret)
                        goto out;
        }

        if (pmsg.dropped) {
                /* our last seen message is gone, return error and reset */
                atomic64_set(&user->seq, pmsg.seq);
                ret = -EPIPE;
                goto out;
        }

        atomic64_set(&user->seq, pmsg.seq + 1);

        if (pmsg.outbuf_len > count) {
                ret = -EINVAL;
                goto out;
        }

        if (copy_to_user(buf, outbuf, pmsg.outbuf_len)) {
                ret = -EFAULT;
                goto out;
        }
        ret = pmsg.outbuf_len;
out:
        mutex_unlock(&user->lock);
        return ret;
}

/*
 * Be careful when modifying this function!!!
 *
 * Only few operations are supported because the device works only with the
 * entire variable length messages (records). Non-standard values are
 * returned in the other cases and has been this way for quite some time.
 * User space applications might depend on this behavior.
 */
static loff_t devkmsg_llseek(struct file *file, loff_t offset, int whence)
{
        struct devkmsg_user *user = file->private_data;
        loff_t ret = 0;

        if (offset)
                return -ESPIPE;

        switch (whence) {
        case SEEK_SET:
                /* the first record */
                atomic64_set(&user->seq, prb_first_valid_seq(prb));
                break;
        case SEEK_DATA:
                /*
                 * The first record after the last SYSLOG_ACTION_CLEAR,
                 * like issued by 'dmesg -c'. Reading /dev/kmsg itself
                 * changes no global state, and does not clear anything.
                 */
                atomic64_set(&user->seq, latched_seq_read_nolock(&clear_seq));
                break;
        case SEEK_END:
                /* after the last record */
                atomic64_set(&user->seq, prb_next_seq(prb));
                break;
        default:
                ret = -EINVAL;
        }
        return ret;
}

static __poll_t devkmsg_poll(struct file *file, poll_table *wait)
{
        struct devkmsg_user *user = file->private_data;
        struct printk_info info;
        __poll_t ret = 0;

        poll_wait(file, &log_wait, wait);

        if (prb_read_valid_info(prb, atomic64_read(&user->seq), &info, NULL)) {
                /* return error when data has vanished underneath us */
                if (info.seq != atomic64_read(&user->seq))
                        ret = EPOLLIN|EPOLLRDNORM|EPOLLERR|EPOLLPRI;
                else
                        ret = EPOLLIN|EPOLLRDNORM;
        }

        return ret;
}

static int devkmsg_open(struct inode *inode, struct file *file)
{
        struct devkmsg_user *user;
        int err;

        if (devkmsg_log & DEVKMSG_LOG_MASK_OFF)
                return -EPERM;

        /* write-only does not need any file context */
        if ((file->f_flags & O_ACCMODE) != O_WRONLY) {
                err = check_syslog_permissions(SYSLOG_ACTION_READ_ALL,
                                               SYSLOG_FROM_READER);
                if (err)
                        return err;
        }

        user = kvmalloc(sizeof(struct devkmsg_user), GFP_KERNEL);
        if (!user)
                return -ENOMEM;

        ratelimit_default_init(&user->rs);
        ratelimit_set_flags(&user->rs, RATELIMIT_MSG_ON_RELEASE);

        mutex_init(&user->lock);

        atomic64_set(&user->seq, prb_first_valid_seq(prb));

        file->private_data = user;
        return 0;
}

static int devkmsg_release(struct inode *inode, struct file *file)
{
        struct devkmsg_user *user = file->private_data;

        ratelimit_state_exit(&user->rs);

        mutex_destroy(&user->lock);
        kvfree(user);
        return 0;
}

const struct file_operations kmsg_fops = {
        .open = devkmsg_open,
        .read = devkmsg_read,
        .write_iter = devkmsg_write,
        .llseek = devkmsg_llseek,
        .poll = devkmsg_poll,
        .release = devkmsg_release,
};

#ifdef CONFIG_VMCORE_INFO
/*
 * This appends the listed symbols to /proc/vmcore
 *
 * /proc/vmcore is used by various utilities, like crash and makedumpfile to
 * obtain access to symbols that are otherwise very difficult to locate.  These
 * symbols are specifically used so that utilities can access and extract the
 * dmesg log from a vmcore file after a crash.
 */
void log_buf_vmcoreinfo_setup(void)
{
        struct dev_printk_info *dev_info = NULL;

        VMCOREINFO_SYMBOL(prb);
        VMCOREINFO_SYMBOL(printk_rb_static);
        VMCOREINFO_SYMBOL(clear_seq);

        /*
         * Export struct size and field offsets. User space tools can
         * parse it and detect any changes to structure down the line.
         */

        VMCOREINFO_STRUCT_SIZE(printk_ringbuffer);
        VMCOREINFO_OFFSET(printk_ringbuffer, desc_ring);
        VMCOREINFO_OFFSET(printk_ringbuffer, text_data_ring);
        VMCOREINFO_OFFSET(printk_ringbuffer, fail);

        VMCOREINFO_STRUCT_SIZE(prb_desc_ring);
        VMCOREINFO_OFFSET(prb_desc_ring, count_bits);
        VMCOREINFO_OFFSET(prb_desc_ring, descs);
        VMCOREINFO_OFFSET(prb_desc_ring, infos);
        VMCOREINFO_OFFSET(prb_desc_ring, head_id);
        VMCOREINFO_OFFSET(prb_desc_ring, tail_id);

        VMCOREINFO_STRUCT_SIZE(prb_desc);
        VMCOREINFO_OFFSET(prb_desc, state_var);
        VMCOREINFO_OFFSET(prb_desc, text_blk_lpos);

        VMCOREINFO_STRUCT_SIZE(prb_data_blk_lpos);
        VMCOREINFO_OFFSET(prb_data_blk_lpos, begin);
        VMCOREINFO_OFFSET(prb_data_blk_lpos, next);

        VMCOREINFO_STRUCT_SIZE(printk_info);
        VMCOREINFO_OFFSET(printk_info, seq);
        VMCOREINFO_OFFSET(printk_info, ts_nsec);
        VMCOREINFO_OFFSET(printk_info, text_len);
        VMCOREINFO_OFFSET(printk_info, caller_id);
        VMCOREINFO_OFFSET(printk_info, dev_info);

        VMCOREINFO_STRUCT_SIZE(dev_printk_info);
        VMCOREINFO_OFFSET(dev_printk_info, subsystem);
        VMCOREINFO_LENGTH(printk_info_subsystem, sizeof(dev_info->subsystem));
        VMCOREINFO_OFFSET(dev_printk_info, device);
        VMCOREINFO_LENGTH(printk_info_device, sizeof(dev_info->device));

        VMCOREINFO_STRUCT_SIZE(prb_data_ring);
        VMCOREINFO_OFFSET(prb_data_ring, size_bits);
        VMCOREINFO_OFFSET(prb_data_ring, data);
        VMCOREINFO_OFFSET(prb_data_ring, head_lpos);
        VMCOREINFO_OFFSET(prb_data_ring, tail_lpos);

        VMCOREINFO_SIZE(atomic_long_t);
        VMCOREINFO_TYPE_OFFSET(atomic_long_t, counter);

        VMCOREINFO_STRUCT_SIZE(latched_seq);
        VMCOREINFO_OFFSET(latched_seq, val);
}
#endif

/* requested log_buf_len from kernel cmdline */
static unsigned long __initdata new_log_buf_len;

/* we practice scaling the ring buffer by powers of 2 */
static void __init log_buf_len_update(u64 size)
{
        if (size > (u64)LOG_BUF_LEN_MAX) {
                size = (u64)LOG_BUF_LEN_MAX;
                pr_err("log_buf over 2G is not supported.\n");
        }

        if (size)
                size = roundup_pow_of_two(size);
        if (size > log_buf_len)
                new_log_buf_len = (unsigned long)size;
}

/* save requested log_buf_len since it's too early to process it */
static int __init log_buf_len_setup(char *str)
{
        u64 size;

        if (!str)
                return -EINVAL;

        size = memparse(str, &str);

        log_buf_len_update(size);

        return 0;
}
early_param("log_buf_len", log_buf_len_setup);

#ifdef CONFIG_SMP
#define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT)

static void __init log_buf_add_cpu(void)
{
        unsigned int cpu_extra;

        /*
         * archs should set up cpu_possible_bits properly with
         * set_cpu_possible() after setup_arch() but just in
         * case lets ensure this is valid.
         */
        if (num_possible_cpus() == 1)
                return;

        cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN;

        /* by default this will only continue through for large > 64 CPUs */
        if (cpu_extra <= __LOG_BUF_LEN / 2)
                return;

        pr_info("log_buf_len individual max cpu contribution: %d bytes\n",
                __LOG_CPU_MAX_BUF_LEN);
        pr_info("log_buf_len total cpu_extra contributions: %d bytes\n",
                cpu_extra);
        pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN);

        log_buf_len_update(cpu_extra + __LOG_BUF_LEN);
}
#else /* !CONFIG_SMP */
static inline void log_buf_add_cpu(void) {}
#endif /* CONFIG_SMP */

static void __init set_percpu_data_ready(void)
{
        __printk_percpu_data_ready = true;
}

static unsigned int __init add_to_rb(struct printk_ringbuffer *rb,
                                     struct printk_record *r)
{
        struct prb_reserved_entry e;
        struct printk_record dest_r;

        prb_rec_init_wr(&dest_r, r->info->text_len);

        if (!prb_reserve(&e, rb, &dest_r))
                return 0;

        memcpy(&dest_r.text_buf[0], &r->text_buf[0], r->info->text_len);
        dest_r.info->text_len = r->info->text_len;
        dest_r.info->facility = r->info->facility;
        dest_r.info->level = r->info->level;
        dest_r.info->flags = r->info->flags;
        dest_r.info->ts_nsec = r->info->ts_nsec;
        dest_r.info->caller_id = r->info->caller_id;
        memcpy(&dest_r.info->dev_info, &r->info->dev_info, sizeof(dest_r.info->dev_info));

        prb_final_commit(&e);

        return prb_record_text_space(&e);
}

static char setup_text_buf[PRINTKRB_RECORD_MAX] __initdata;

static void print_log_buf_usage_stats(void)
{
        unsigned int descs_count = log_buf_len >> PRB_AVGBITS;
        size_t meta_data_size;

        meta_data_size = descs_count * (sizeof(struct prb_desc) + sizeof(struct printk_info));

        pr_info("log buffer data + meta data: %u + %zu = %zu bytes\n",
                log_buf_len, meta_data_size, log_buf_len + meta_data_size);
}

void __init setup_log_buf(int early)
{
        struct printk_info *new_infos;
        unsigned int new_descs_count;
        struct prb_desc *new_descs;
        struct printk_info info;
        struct printk_record r;
        unsigned int text_size;
        size_t new_descs_size;
        size_t new_infos_size;
        unsigned long flags;
        char *new_log_buf;
        unsigned int free;
        u64 seq;

        /*
         * Some archs call setup_log_buf() multiple times - first is very
         * early, e.g. from setup_arch(), and second - when percpu_areas
         * are initialised.
         */
        if (!early)
                set_percpu_data_ready();

        if (log_buf != __log_buf)
                return;

        if (!early && !new_log_buf_len)
                log_buf_add_cpu();

        if (!new_log_buf_len) {
                /* Show the memory stats only once. */
                if (!early)
                        goto out;

                return;
        }

        new_descs_count = new_log_buf_len >> PRB_AVGBITS;
        if (new_descs_count == 0) {
                pr_err("new_log_buf_len: %lu too small\n", new_log_buf_len);
                goto out;
        }

        new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN);
        if (unlikely(!new_log_buf)) {
                pr_err("log_buf_len: %lu text bytes not available\n",
                       new_log_buf_len);
                goto out;
        }

        new_descs_size = new_descs_count * sizeof(struct prb_desc);
        new_descs = memblock_alloc(new_descs_size, LOG_ALIGN);
        if (unlikely(!new_descs)) {
                pr_err("log_buf_len: %zu desc bytes not available\n",
                       new_descs_size);
                goto err_free_log_buf;
        }

        new_infos_size = new_descs_count * sizeof(struct printk_info);
        new_infos = memblock_alloc(new_infos_size, LOG_ALIGN);
        if (unlikely(!new_infos)) {
                pr_err("log_buf_len: %zu info bytes not available\n",
                       new_infos_size);
                goto err_free_descs;
        }

        prb_rec_init_rd(&r, &info, &setup_text_buf[0], sizeof(setup_text_buf));

        prb_init(&printk_rb_dynamic,
                 new_log_buf, ilog2(new_log_buf_len),
                 new_descs, ilog2(new_descs_count),
                 new_infos);

        local_irq_save(flags);

        log_buf_len = new_log_buf_len;
        log_buf = new_log_buf;
        new_log_buf_len = 0;

        free = __LOG_BUF_LEN;
        prb_for_each_record(0, &printk_rb_static, seq, &r) {
                text_size = add_to_rb(&printk_rb_dynamic, &r);
                if (text_size > free)
                        free = 0;
                else
                        free -= text_size;
        }

        prb = &printk_rb_dynamic;

        local_irq_restore(flags);

        /*
         * Copy any remaining messages that might have appeared from
         * NMI context after copying but before switching to the
         * dynamic buffer.
         */
        prb_for_each_record(seq, &printk_rb_static, seq, &r) {
                text_size = add_to_rb(&printk_rb_dynamic, &r);
                if (text_size > free)
                        free = 0;
                else
                        free -= text_size;
        }

        if (seq != prb_next_seq(&printk_rb_static)) {
                pr_err("dropped %llu messages\n",
                       prb_next_seq(&printk_rb_static) - seq);
        }

        print_log_buf_usage_stats();
        pr_info("early log buf free: %u(%u%%)\n",
                free, (free * 100) / __LOG_BUF_LEN);
        return;

err_free_descs:
        memblock_free(new_descs, new_descs_size);
err_free_log_buf:
        memblock_free(new_log_buf, new_log_buf_len);
out:
        print_log_buf_usage_stats();
}

static bool __read_mostly ignore_loglevel;

static int __init ignore_loglevel_setup(char *str)
{
        ignore_loglevel = true;
        pr_info("debug: ignoring loglevel setting.\n");

        return 0;
}

early_param("ignore_loglevel", ignore_loglevel_setup);
module_param(ignore_loglevel, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(ignore_loglevel,
                 "ignore loglevel setting (prints all kernel messages to the console)");

static bool suppress_message_printing(int level)
{
        return (level >= console_loglevel && !ignore_loglevel);
}

#ifdef CONFIG_BOOT_PRINTK_DELAY

static int boot_delay; /* msecs delay after each printk during bootup */
static unsigned long long loops_per_msec;        /* based on boot_delay */

static int __init boot_delay_setup(char *str)
{
        unsigned long lpj;

        lpj = preset_lpj ? preset_lpj : 1000000;        /* some guess */
        loops_per_msec = (unsigned long long)lpj / 1000 * HZ;

        get_option(&str, &boot_delay);
        if (boot_delay > 10 * 1000)
                boot_delay = 0;

        pr_debug("boot_delay: %u, preset_lpj: %ld, lpj: %lu, "
                "HZ: %d, loops_per_msec: %llu\n",
                boot_delay, preset_lpj, lpj, HZ, loops_per_msec);
        return 0;
}
early_param("boot_delay", boot_delay_setup);

static void boot_delay_msec(int level)
{
        unsigned long long k;
        unsigned long timeout;
        bool suppress = !is_printk_force_console() &&
                        suppress_message_printing(level);

        if ((boot_delay == 0 || system_state >= SYSTEM_RUNNING) || suppress)
                return;

        k = (unsigned long long)loops_per_msec * boot_delay;

        timeout = jiffies + msecs_to_jiffies(boot_delay);
        while (k) {
                k--;
                cpu_relax();
                /*
                 * use (volatile) jiffies to prevent
                 * compiler reduction; loop termination via jiffies
                 * is secondary and may or may not happen.
                 */
                if (time_after(jiffies, timeout))
                        break;
                touch_nmi_watchdog();
        }
}
#else
static inline void boot_delay_msec(int level)
{
}
#endif

static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME);
module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR);

static size_t print_syslog(unsigned int level, char *buf)
{
        return sprintf(buf, "<%u>", level);
}

static size_t print_time(u64 ts, char *buf)
{
        unsigned long rem_nsec = do_div(ts, 1000000000);

        return sprintf(buf, "[%5lu.%06lu]",
                       (unsigned long)ts, rem_nsec / 1000);
}

#ifdef CONFIG_PRINTK_CALLER
static size_t print_caller(u32 id, char *buf)
{
        char caller[12];

        snprintf(caller, sizeof(caller), "%c%u",
                 id & 0x80000000 ? 'C' : 'T', id & ~0x80000000);
        return sprintf(buf, "[%6s]", caller);
}
#else
#define print_caller(id, buf) 0
#endif

static size_t info_print_prefix(const struct printk_info  *info, bool syslog,
                                bool time, char *buf)
{
        size_t len = 0;

        if (syslog)
                len = print_syslog((info->facility << 3) | info->level, buf);

        if (time)
                len += print_time(info->ts_nsec, buf + len);

        len += print_caller(info->caller_id, buf + len);

        if (IS_ENABLED(CONFIG_PRINTK_CALLER) || time) {
                buf[len++] = ' ';
                buf[len] = '\0';
        }

        return len;
}

/*
 * Prepare the record for printing. The text is shifted within the given
 * buffer to avoid a need for another one. The following operations are
 * done:
 *
 *   - Add prefix for each line.
 *   - Drop truncated lines that no longer fit into the buffer.
 *   - Add the trailing newline that has been removed in vprintk_store().
 *   - Add a string terminator.
 *
 * Since the produced string is always terminated, the maximum possible
 * return value is @r->text_buf_size - 1;
 *
 * Return: The length of the updated/prepared text, including the added
 * prefixes and the newline. The terminator is not counted. The dropped
 * line(s) are not counted.
 */
static size_t record_print_text(struct printk_record *r, bool syslog,
                                bool time)
{
        size_t text_len = r->info->text_len;
        size_t buf_size = r->text_buf_size;
        char *text = r->text_buf;
        char prefix[PRINTK_PREFIX_MAX];
        bool truncated = false;
        size_t prefix_len;
        size_t line_len;
        size_t len = 0;
        char *next;

        /*
         * If the message was truncated because the buffer was not large
         * enough, treat the available text as if it were the full text.
         */
        if (text_len > buf_size)
                text_len = buf_size;

        prefix_len = info_print_prefix(r->info, syslog, time, prefix);

        /*
         * @text_len: bytes of unprocessed text
         * @line_len: bytes of current line _without_ newline
         * @text:     pointer to beginning of current line
         * @len:      number of bytes prepared in r->text_buf
         */
        for (;;) {
                next = memchr(text, '\n', text_len);
                if (next) {
                        line_len = next - text;
                } else {
                        /* Drop truncated line(s). */
                        if (truncated)
                                break;
                        line_len = text_len;
                }

                /*
                 * Truncate the text if there is not enough space to add the
                 * prefix and a trailing newline and a terminator.
                 */
                if (len + prefix_len + text_len + 1 + 1 > buf_size) {
                        /* Drop even the current line if no space. */
                        if (len + prefix_len + line_len + 1 + 1 > buf_size)
                                break;

                        text_len = buf_size - len - prefix_len - 1 - 1;
                        truncated = true;
                }

                memmove(text + prefix_len, text, text_len);
                memcpy(text, prefix, prefix_len);

                /*
                 * Increment the prepared length to include the text and
                 * prefix that were just moved+copied. Also increment for the
                 * newline at the end of this line. If this is the last line,
                 * there is no newline, but it will be added immediately below.
                 */
                len += prefix_len + line_len + 1;
                if (text_len == line_len) {
                        /*
                         * This is the last line. Add the trailing newline
                         * removed in vprintk_store().
                         */
                        text[prefix_len + line_len] = '\n';
                        break;
                }

                /*
                 * Advance beyond the added prefix and the related line with
                 * its newline.
                 */
                text += prefix_len + line_len + 1;

                /*
                 * The remaining text has only decreased by the line with its
                 * newline.
                 *
                 * Note that @text_len can become zero. It happens when @text
                 * ended with a newline (either due to truncation or the
                 * original string ending with "\n\n"). The loop is correctly
                 * repeated and (if not truncated) an empty line with a prefix
                 * will be prepared.
                 */
                text_len -= line_len + 1;
        }

        /*
         * If a buffer was provided, it will be terminated. Space for the
         * string terminator is guaranteed to be available. The terminator is
         * not counted in the return value.
         */
        if (buf_size > 0)
                r->text_buf[len] = 0;

        return len;
}

static size_t get_record_print_text_size(struct printk_info *info,
                                         unsigned int line_count,
                                         bool syslog, bool time)
{
        char prefix[PRINTK_PREFIX_MAX];
        size_t prefix_len;

        prefix_len = info_print_prefix(info, syslog, time, prefix);

        /*
         * Each line will be preceded with a prefix. The intermediate
         * newlines are already within the text, but a final trailing
         * newline will be added.
         */
        return ((prefix_len * line_count) + info->text_len + 1);
}

/*
 * Beginning with @start_seq, find the first record where it and all following
 * records up to (but not including) @max_seq fit into @size.
 *
 * @max_seq is simply an upper bound and does not need to exist. If the caller
 * does not require an upper bound, -1 can be used for @max_seq.
 */
static u64 find_first_fitting_seq(u64 start_seq, u64 max_seq, size_t size,
                                  bool syslog, bool time)
{
        struct printk_info info;
        unsigned int line_count;
        size_t len = 0;
        u64 seq;

        /* Determine the size of the records up to @max_seq. */
        prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
                if (info.seq >= max_seq)
                        break;
                len += get_record_print_text_size(&info, line_count, syslog, time);
        }

        /*
         * Adjust the upper bound for the next loop to avoid subtracting
         * lengths that were never added.
         */
        if (seq < max_seq)
                max_seq = seq;

        /*
         * Move first record forward until length fits into the buffer. Ignore
         * newest messages that were not counted in the above cycle. Messages
         * might appear and get lost in the meantime. This is a best effort
         * that prevents an infinite loop that could occur with a retry.
         */
        prb_for_each_info(start_seq, prb, seq, &info, &line_count) {
                if (len <= size || info.seq >= max_seq)
                        break;
                len -= get_record_print_text_size(&info, line_count, syslog, time);
        }

        return seq;
}

/* The caller is responsible for making sure @size is greater than 0. */
static int syslog_print(char __user *buf, int size)
{
        struct printk_info info;
        struct printk_record r;
        char *text;
        int len = 0;
        u64 seq;

        text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
        if (!text)
                return -ENOMEM;

        prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);

        mutex_lock(&syslog_lock);

        /*
         * Wait for the @syslog_seq record to be available. @syslog_seq may
         * change while waiting.
         */
        do {
                seq = syslog_seq;

                mutex_unlock(&syslog_lock);
                /*
                 * Guarantee this task is visible on the waitqueue before
                 * checking the wake condition.
                 *
                 * The full memory barrier within set_current_state() of
                 * prepare_to_wait_event() pairs with the full memory barrier
                 * within wq_has_sleeper().
                 *
                 * This pairs with __wake_up_klogd:A.
                 */
                len = wait_event_interruptible(log_wait,
                                prb_read_valid(prb, seq, NULL)); /* LMM(syslog_print:A) */
                mutex_lock(&syslog_lock);

                if (len)
                        goto out;
        } while (syslog_seq != seq);

        /*
         * Copy records that fit into the buffer. The above cycle makes sure
         * that the first record is always available.
         */
        do {
                size_t n;
                size_t skip;
                int err;

                if (!prb_read_valid(prb, syslog_seq, &r))
                        break;

                if (r.info->seq != syslog_seq) {
                        /* message is gone, move to next valid one */
                        syslog_seq = r.info->seq;
                        syslog_partial = 0;
                }

                /*
                 * To keep reading/counting partial line consistent,
                 * use printk_time value as of the beginning of a line.
                 */
                if (!syslog_partial)
                        syslog_time = printk_time;

                skip = syslog_partial;
                n = record_print_text(&r, true, syslog_time);
                if (n - syslog_partial <= size) {
                        /* message fits into buffer, move forward */
                        syslog_seq = r.info->seq + 1;
                        n -= syslog_partial;
                        syslog_partial = 0;
                } else if (!len){
                        /* partial read(), remember position */
                        n = size;
                        syslog_partial += n;
                } else
                        n = 0;

                if (!n)
                        break;

                mutex_unlock(&syslog_lock);
                err = copy_to_user(buf, text + skip, n);
                mutex_lock(&syslog_lock);

                if (err) {
                        if (!len)
                                len = -EFAULT;
                        break;
                }

                len += n;
                size -= n;
                buf += n;
        } while (size);
out:
        mutex_unlock(&syslog_lock);
        kfree(text);
        return len;
}

static int syslog_print_all(char __user *buf, int size, bool clear)
{
        struct printk_info info;
        struct printk_record r;
        char *text;
        int len = 0;
        u64 seq;
        bool time;

        text = kmalloc(PRINTK_MESSAGE_MAX, GFP_KERNEL);
        if (!text)
                return -ENOMEM;

        time = printk_time;
        /*
         * Find first record that fits, including all following records,
         * into the user-provided buffer for this dump.
         */
        seq = find_first_fitting_seq(latched_seq_read_nolock(&clear_seq), -1,
                                     size, true, time);

        prb_rec_init_rd(&r, &info, text, PRINTK_MESSAGE_MAX);

        prb_for_each_record(seq, prb, seq, &r) {
                int textlen;

                textlen = record_print_text(&r, true, time);

                if (len + textlen > size) {
                        seq--;
                        break;
                }

                if (copy_to_user(buf + len, text, textlen))
                        len = -EFAULT;
                else
                        len += textlen;

                if (len < 0)
                        break;
        }

        if (clear) {
                mutex_lock(&syslog_lock);
                latched_seq_write(&clear_seq, seq);
                mutex_unlock(&syslog_lock);
        }

        kfree(text);
        return len;
}

static void syslog_clear(void)
{
        mutex_lock(&syslog_lock);
        latched_seq_write(&clear_seq, prb_next_seq(prb));
        mutex_unlock(&syslog_lock);
}

int do_syslog(int type, char __user *buf, int len, int source)
{
        struct printk_info info;
        bool clear = false;
        static int saved_console_loglevel = LOGLEVEL_DEFAULT;
        int error;

        error = check_syslog_permissions(type, source);
        if (error)
                return error;

        switch (type) {
        case SYSLOG_ACTION_CLOSE:        /* Close log */
                break;
        case SYSLOG_ACTION_OPEN:        /* Open log */
                break;
        case SYSLOG_ACTION_READ:        /* Read from log */
                if (!buf || len < 0)
                        return -EINVAL;
                if (!len)
                        return 0;
                if (!access_ok(buf, len))
                        return -EFAULT;
                error = syslog_print(buf, len);
                break;
        /* Read/clear last kernel messages */
        case SYSLOG_ACTION_READ_CLEAR:
                clear = true;
                fallthrough;
        /* Read last kernel messages */
        case SYSLOG_ACTION_READ_ALL:
                if (!buf || len < 0)
                        return -EINVAL;
                if (!len)
                        return 0;
                if (!access_ok(buf, len))
                        return -EFAULT;
                error = syslog_print_all(buf, len, clear);
                break;
        /* Clear ring buffer */
        case SYSLOG_ACTION_CLEAR:
                syslog_clear();
                break;
        /* Disable logging to console */
        case SYSLOG_ACTION_CONSOLE_OFF:
                if (saved_console_loglevel == LOGLEVEL_DEFAULT)
                        saved_console_loglevel = console_loglevel;
                console_loglevel = minimum_console_loglevel;
                break;
        /* Enable logging to console */
        case SYSLOG_ACTION_CONSOLE_ON:
                if (saved_console_loglevel != LOGLEVEL_DEFAULT) {
                        console_loglevel = saved_console_loglevel;
                        saved_console_loglevel = LOGLEVEL_DEFAULT;
                }
                break;
        /* Set level of messages printed to console */
        case SYSLOG_ACTION_CONSOLE_LEVEL:
                if (len < 1 || len > 8)
                        return -EINVAL;
                if (len < minimum_console_loglevel)
                        len = minimum_console_loglevel;
                console_loglevel = len;
                /* Implicitly re-enable logging to console */
                saved_console_loglevel = LOGLEVEL_DEFAULT;
                break;
        /* Number of chars in the log buffer */
        case SYSLOG_ACTION_SIZE_UNREAD:
                mutex_lock(&syslog_lock);
                if (!prb_read_valid_info(prb, syslog_seq, &info, NULL)) {
                        /* No unread messages. */
                        mutex_unlock(&syslog_lock);
                        return 0;
                }
                if (info.seq != syslog_seq) {
                        /* messages are gone, move to first one */
                        syslog_seq = info.seq;
                        syslog_partial = 0;
                }
                if (source == SYSLOG_FROM_PROC) {
                        /*
                         * Short-cut for poll(/"proc/kmsg") which simply checks
                         * for pending data, not the size; return the count of
                         * records, not the length.
                         */
                        error = prb_next_seq(prb) - syslog_seq;
                } else {
                        bool time = syslog_partial ? syslog_time : printk_time;
                        unsigned int line_count;
                        u64 seq;

                        prb_for_each_info(syslog_seq, prb, seq, &info,
                                          &line_count) {
                                error += get_record_print_text_size(&info, line_count,
                                                                    true, time);
                                time = printk_time;
                        }
                        error -= syslog_partial;
                }
                mutex_unlock(&syslog_lock);
                break;
        /* Size of the log buffer */
        case SYSLOG_ACTION_SIZE_BUFFER:
                error = log_buf_len;
                break;
        default:
                error = -EINVAL;
                break;
        }

        return error;
}

SYSCALL_DEFINE3(syslog, int, type, char __user *, buf, int, len)
{
        return do_syslog(type, buf, len, SYSLOG_FROM_READER);
}

/*
 * Special console_lock variants that help to reduce the risk of soft-lockups.
 * They allow to pass console_lock to another printk() call using a busy wait.
 */

#ifdef CONFIG_LOCKDEP
static struct lockdep_map console_owner_dep_map = {
        .name = "console_owner"
};
#endif

static DEFINE_RAW_SPINLOCK(console_owner_lock);
static struct task_struct *console_owner;
static bool console_waiter;

/**
 * console_lock_spinning_enable - mark beginning of code where another
 *        thread might safely busy wait
 *
 * This basically converts console_lock into a spinlock. This marks
 * the section where the console_lock owner can not sleep, because
 * there may be a waiter spinning (like a spinlock). Also it must be
 * ready to hand over the lock at the end of the section.
 */
void console_lock_spinning_enable(void)
{
        /*
         * Do not use spinning in panic(). The panic CPU wants to keep the lock.
         * Non-panic CPUs abandon the flush anyway.
         *
         * Just keep the lockdep annotation. The panic-CPU should avoid
         * taking console_owner_lock because it might cause a deadlock.
         * This looks like the easiest way how to prevent false lockdep
         * reports without handling races a lockless way.
         */
        if (panic_in_progress())
                goto lockdep;

        raw_spin_lock(&console_owner_lock);
        console_owner = current;
        raw_spin_unlock(&console_owner_lock);

lockdep:
        /* The waiter may spin on us after setting console_owner */
        spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
}

/**
 * console_lock_spinning_disable_and_check - mark end of code where another
 *        thread was able to busy wait and check if there is a waiter
 * @cookie: cookie returned from console_srcu_read_lock()
 *
 * This is called at the end of the section where spinning is allowed.
 * It has two functions. First, it is a signal that it is no longer
 * safe to start busy waiting for the lock. Second, it checks if
 * there is a busy waiter and passes the lock rights to her.
 *
 * Important: Callers lose both the console_lock and the SRCU read lock if
 *        there was a busy waiter. They must not touch items synchronized by
 *        console_lock or SRCU read lock in this case.
 *
 * Return: 1 if the lock rights were passed, 0 otherwise.
 */
int console_lock_spinning_disable_and_check(int cookie)
{
        int waiter;

        /*
         * Ignore spinning waiters during panic() because they might get stopped
         * or blocked at any time,
         *
         * It is safe because nobody is allowed to start spinning during panic
         * in the first place. If there has been a waiter then non panic CPUs
         * might stay spinning. They would get stopped anyway. The panic context
         * will never start spinning and an interrupted spin on panic CPU will
         * never continue.
         */
        if (panic_in_progress()) {
                /* Keep lockdep happy. */
                spin_release(&console_owner_dep_map, _THIS_IP_);
                return 0;
        }

        raw_spin_lock(&console_owner_lock);
        waiter = READ_ONCE(console_waiter);
        console_owner = NULL;
        raw_spin_unlock(&console_owner_lock);

        if (!waiter) {
                spin_release(&console_owner_dep_map, _THIS_IP_);
                return 0;
        }

        /* The waiter is now free to continue */
        WRITE_ONCE(console_waiter, false);

        spin_release(&console_owner_dep_map, _THIS_IP_);

        /*
         * Preserve lockdep lock ordering. Release the SRCU read lock before
         * releasing the console_lock.
         */
        console_srcu_read_unlock(cookie);

        /*
         * Hand off console_lock to waiter. The waiter will perform
         * the up(). After this, the waiter is the console_lock owner.
         */
        mutex_release(&console_lock_dep_map, _THIS_IP_);
        return 1;
}

/**
 * console_trylock_spinning - try to get console_lock by busy waiting
 *
 * This allows to busy wait for the console_lock when the current
 * owner is running in specially marked sections. It means that
 * the current owner is running and cannot reschedule until it
 * is ready to lose the lock.
 *
 * Return: 1 if we got the lock, 0 othrewise
 */
static int console_trylock_spinning(void)
{
        struct task_struct *owner = NULL;
        bool waiter;
        bool spin = false;
        unsigned long flags;

        if (console_trylock())
                return 1;

        /*
         * It's unsafe to spin once a panic has begun. If we are the
         * panic CPU, we may have already halted the owner of the
         * console_sem. If we are not the panic CPU, then we should
         * avoid taking console_sem, so the panic CPU has a better
         * chance of cleanly acquiring it later.
         */
        if (panic_in_progress())
                return 0;

        printk_safe_enter_irqsave(flags);

        raw_spin_lock(&console_owner_lock);
        owner = READ_ONCE(console_owner);
        waiter = READ_ONCE(console_waiter);
        if (!waiter && owner && owner != current) {
                WRITE_ONCE(console_waiter, true);
                spin = true;
        }
        raw_spin_unlock(&console_owner_lock);

        /*
         * If there is an active printk() writing to the
         * consoles, instead of having it write our data too,
         * see if we can offload that load from the active
         * printer, and do some printing ourselves.
         * Go into a spin only if there isn't already a waiter
         * spinning, and there is an active printer, and
         * that active printer isn't us (recursive printk?).
         */
        if (!spin) {
                printk_safe_exit_irqrestore(flags);
                return 0;
        }

        /* We spin waiting for the owner to release us */
        spin_acquire(&console_owner_dep_map, 0, 0, _THIS_IP_);
        /* Owner will clear console_waiter on hand off */
        while (READ_ONCE(console_waiter))
                cpu_relax();
        spin_release(&console_owner_dep_map, _THIS_IP_);

        printk_safe_exit_irqrestore(flags);
        /*
         * The owner passed the console lock to us.
         * Since we did not spin on console lock, annotate
         * this as a trylock. Otherwise lockdep will
         * complain.
         */
        mutex_acquire(&console_lock_dep_map, 0, 1, _THIS_IP_);

        /*
         * Update @console_may_schedule for trylock because the previous
         * owner may have been schedulable.
         */
        console_may_schedule = 0;

        return 1;
}

/*
 * Recursion is tracked separately on each CPU. If NMIs are supported, an
 * additional NMI context per CPU is also separately tracked. Until per-CPU
 * is available, a separate "early tracking" is performed.
 */
static DEFINE_PER_CPU(u8, printk_count);
static u8 printk_count_early;
#ifdef CONFIG_HAVE_NMI
static DEFINE_PER_CPU(u8, printk_count_nmi);
static u8 printk_count_nmi_early;
#endif

/*
 * Recursion is limited to keep the output sane. printk() should not require
 * more than 1 level of recursion (allowing, for example, printk() to trigger
 * a WARN), but a higher value is used in case some printk-internal errors
 * exist, such as the ringbuffer validation checks failing.
 */
#define PRINTK_MAX_RECURSION 3

/*
 * Return a pointer to the dedicated counter for the CPU+context of the
 * caller.
 */
static u8 *__printk_recursion_counter(void)
{
#ifdef CONFIG_HAVE_NMI
        if (in_nmi()) {
                if (printk_percpu_data_ready())
                        return this_cpu_ptr(&printk_count_nmi);
                return &printk_count_nmi_early;
        }
#endif
        if (printk_percpu_data_ready())
                return this_cpu_ptr(&printk_count);
        return &printk_count_early;
}

/*
 * Enter recursion tracking. Interrupts are disabled to simplify tracking.
 * The caller must check the boolean return value to see if the recursion is
 * allowed. On failure, interrupts are not disabled.
 *
 * @recursion_ptr must be a variable of type (u8 *) and is the same variable
 * that is passed to printk_exit_irqrestore().
 */
#define printk_enter_irqsave(recursion_ptr, flags)        \
({                                                        \
        bool success = true;                                \
                                                        \
        typecheck(u8 *, recursion_ptr);                        \
        local_irq_save(flags);                                \
        (recursion_ptr) = __printk_recursion_counter();        \
        if (*(recursion_ptr) > PRINTK_MAX_RECURSION) {        \
                local_irq_restore(flags);                \
                success = false;                        \
        } else {                                        \
                (*(recursion_ptr))++;                        \
        }                                                \
        success;                                        \
})

/* Exit recursion tracking, restoring interrupts. */
#define printk_exit_irqrestore(recursion_ptr, flags)        \
        do {                                                \
                typecheck(u8 *, recursion_ptr);                \
                (*(recursion_ptr))--;                        \
                local_irq_restore(flags);                \
        } while (0)

int printk_delay_msec __read_mostly;

static inline void printk_delay(int level)
{
        boot_delay_msec(level);

        if (unlikely(printk_delay_msec)) {
                int m = printk_delay_msec;

                while (m--) {
                        mdelay(1);
                        touch_nmi_watchdog();
                }
        }
}

static inline u32 printk_caller_id(void)
{
        return in_task() ? task_pid_nr(current) :
                0x80000000 + smp_processor_id();
}

/**
 * printk_parse_prefix - Parse level and control flags.
 *
 * @text:     The terminated text message.
 * @level:    A pointer to the current level value, will be updated.
 * @flags:    A pointer to the current printk_info flags, will be updated.
 *
 * @level may be NULL if the caller is not interested in the parsed value.
 * Otherwise the variable pointed to by @level must be set to
 * LOGLEVEL_DEFAULT in order to be updated with the parsed value.
 *
 * @flags may be NULL if the caller is not interested in the parsed value.
 * Otherwise the variable pointed to by @flags will be OR'd with the parsed
 * value.
 *
 * Return: The length of the parsed level and control flags.
 */
u16 printk_parse_prefix(const char *text, int *level,
                        enum printk_info_flags *flags)
{
        u16 prefix_len = 0;
        int kern_level;

        while (*text) {
                kern_level = printk_get_level(text);
                if (!kern_level)
                        break;

                switch (kern_level) {
                case '0' ... '7':
                        if (level && *level == LOGLEVEL_DEFAULT)
                                *level = kern_level - '0';
                        break;
                case 'c':        /* KERN_CONT */
                        if (flags)
                                *flags |= LOG_CONT;
                }

                prefix_len += 2;
                text += 2;
        }

        return prefix_len;
}

__printf(5, 0)
static u16 printk_sprint(char *text, u16 size, int facility,
                         enum printk_info_flags *flags, const char *fmt,
                         va_list args)
{
        u16 text_len;

        text_len = vscnprintf(text, size, fmt, args);

        /* Mark and strip a trailing newline. */
        if (text_len && text[text_len - 1] == '\n') {
                text_len--;
                *flags |= LOG_NEWLINE;
        }

        /* Strip log level and control flags. */
        if (facility == 0) {
                u16 prefix_len;

                prefix_len = printk_parse_prefix(text, NULL, NULL);
                if (prefix_len) {
                        text_len -= prefix_len;
                        memmove(text, text + prefix_len, text_len);
                }
        }

        trace_console(text, text_len);

        return text_len;
}

__printf(4, 0)
int vprintk_store(int facility, int level,
                  const struct dev_printk_info *dev_info,
                  const char *fmt, va_list args)
{
        struct prb_reserved_entry e;
        enum printk_info_flags flags = 0;
        struct printk_record r;
        unsigned long irqflags;
        u16 trunc_msg_len = 0;
        char prefix_buf[8];
        u8 *recursion_ptr;
        u16 reserve_size;
        va_list args2;
        u32 caller_id;
        u16 text_len;
        int ret = 0;
        u64 ts_nsec;

        if (!printk_enter_irqsave(recursion_ptr, irqflags))
                return 0;

        /*
         * Since the duration of printk() can vary depending on the message
         * and state of the ringbuffer, grab the timestamp now so that it is
         * close to the call of printk(). This provides a more deterministic
         * timestamp with respect to the caller.
         */
        ts_nsec = local_clock();

        caller_id = printk_caller_id();

        /*
         * The sprintf needs to come first since the syslog prefix might be
         * passed in as a parameter. An extra byte must be reserved so that
         * later the vscnprintf() into the reserved buffer has room for the
         * terminating '\0', which is not counted by vsnprintf().
         */
        va_copy(args2, args);
        reserve_size = vsnprintf(&prefix_buf[0], sizeof(prefix_buf), fmt, args2) + 1;
        va_end(args2);

        if (reserve_size > PRINTKRB_RECORD_MAX)
                reserve_size = PRINTKRB_RECORD_MAX;

        /* Extract log level or control flags. */
        if (facility == 0)
                printk_parse_prefix(&prefix_buf[0], &level, &flags);

        if (level == LOGLEVEL_DEFAULT)
                level = default_message_loglevel;

        if (dev_info)
                flags |= LOG_NEWLINE;

        if (is_printk_force_console())
                flags |= LOG_FORCE_CON;

        if (flags & LOG_CONT) {
                prb_rec_init_wr(&r, reserve_size);
                if (prb_reserve_in_last(&e, prb, &r, caller_id, PRINTKRB_RECORD_MAX)) {
                        text_len = printk_sprint(&r.text_buf[r.info->text_len], reserve_size,
                                                 facility, &flags, fmt, args);
                        r.info->text_len += text_len;

                        if (flags & LOG_FORCE_CON)
                                r.info->flags |= LOG_FORCE_CON;

                        if (flags & LOG_NEWLINE) {
                                r.info->flags |= LOG_NEWLINE;
                                prb_final_commit(&e);
                        } else {
                                prb_commit(&e);
                        }

                        ret = text_len;
                        goto out;
                }
        }

        /*
         * Explicitly initialize the record before every prb_reserve() call.
         * prb_reserve_in_last() and prb_reserve() purposely invalidate the
         * structure when they fail.
         */
        prb_rec_init_wr(&r, reserve_size);
        if (!prb_reserve(&e, prb, &r)) {
                /* truncate the message if it is too long for empty buffer */
                truncate_msg(&reserve_size, &trunc_msg_len);

                prb_rec_init_wr(&r, reserve_size + trunc_msg_len);
                if (!prb_reserve(&e, prb, &r))
                        goto out;
        }

        /* fill message */
        text_len = printk_sprint(&r.text_buf[0], reserve_size, facility, &flags, fmt, args);
        if (trunc_msg_len)
                memcpy(&r.text_buf[text_len], trunc_msg, trunc_msg_len);
        r.info->text_len = text_len + trunc_msg_len;
        r.info->facility = facility;
        r.info->level = level & 7;
        r.info->flags = flags & 0x1f;
        r.info->ts_nsec = ts_nsec;
        r.info->caller_id = caller_id;
        if (dev_info)
                memcpy(&r.info->dev_info, dev_info, sizeof(r.info->dev_info));

        /* A message without a trailing newline can be continued. */
        if (!(flags & LOG_NEWLINE))
                prb_commit(&e);
        else
                prb_final_commit(&e);

        ret = text_len + trunc_msg_len;
out:
        printk_exit_irqrestore(recursion_ptr, irqflags);
        return ret;
}

/*
 * This acts as a one-way switch to allow legacy consoles to print from
 * the printk() caller context on a panic CPU. It also attempts to flush
 * the legacy consoles in this context.
 */
void printk_legacy_allow_panic_sync(void)
{
        struct console_flush_type ft;

        legacy_allow_panic_sync = true;

        printk_get_console_flush_type(&ft);
        if (ft.legacy_direct) {
                if (console_trylock())
                        console_unlock();
        }
}

bool __read_mostly debug_non_panic_cpus;

#ifdef CONFIG_PRINTK_CALLER
static int __init debug_non_panic_cpus_setup(char *str)
{
        debug_non_panic_cpus = true;
        pr_info("allow messages from non-panic CPUs in panic()\n");

        return 0;
}
early_param("debug_non_panic_cpus", debug_non_panic_cpus_setup);
module_param(debug_non_panic_cpus, bool, 0644);
MODULE_PARM_DESC(debug_non_panic_cpus,
                 "allow messages from non-panic CPUs in panic()");
#endif

asmlinkage int vprintk_emit(int facility, int level,
                            const struct dev_printk_info *dev_info,
                            const char *fmt, va_list args)
{
        struct console_flush_type ft;
        int printed_len;

        /* Suppress unimportant messages after panic happens */
        if (unlikely(suppress_printk))
                return 0;

        /*
         * The messages on the panic CPU are the most important. If
         * non-panic CPUs are generating any messages, they will be
         * silently dropped.
         */
        if (other_cpu_in_panic() &&
            !debug_non_panic_cpus &&
            !panic_triggering_all_cpu_backtrace)
                return 0;

        printk_get_console_flush_type(&ft);

        /* If called from the scheduler, we can not call up(). */
        if (level == LOGLEVEL_SCHED) {
                level = LOGLEVEL_DEFAULT;
                ft.legacy_offload |= ft.legacy_direct;
                ft.legacy_direct = false;
        }

        printk_delay(level);

        printed_len = vprintk_store(facility, level, dev_info, fmt, args);

        if (ft.nbcon_atomic)
                nbcon_atomic_flush_pending();

        if (ft.nbcon_offload)
                nbcon_kthreads_wake();

        if (ft.legacy_direct) {
                /*
                 * The caller may be holding system-critical or
                 * timing-sensitive locks. Disable preemption during
                 * printing of all remaining records to all consoles so that
                 * this context can return as soon as possible. Hopefully
                 * another printk() caller will take over the printing.
                 */
                preempt_disable();
                /*
                 * Try to acquire and then immediately release the console
                 * semaphore. The release will print out buffers. With the
                 * spinning variant, this context tries to take over the
                 * printing from another printing context.
                 */
                if (console_trylock_spinning())
                        console_unlock();
                preempt_enable();
        }

        if (ft.legacy_offload)
                defer_console_output();
        else
                wake_up_klogd();

        return printed_len;
}
EXPORT_SYMBOL(vprintk_emit);

int vprintk_default(const char *fmt, va_list args)
{
        return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, fmt, args);
}
EXPORT_SYMBOL_GPL(vprintk_default);

asmlinkage __visible int _printk(const char *fmt, ...)
{
        va_list args;
        int r;

        va_start(args, fmt);
        r = vprintk(fmt, args);
        va_end(args);

        return r;
}
EXPORT_SYMBOL(_printk);

static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress);

#else /* CONFIG_PRINTK */

#define printk_time                false

#define prb_read_valid(rb, seq, r)        false
#define prb_first_valid_seq(rb)                0
#define prb_next_seq(rb)                0

static u64 syslog_seq;

static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress) { return true; }

#endif /* CONFIG_PRINTK */

#ifdef CONFIG_EARLY_PRINTK
struct console *early_console;

asmlinkage __visible void early_printk(const char *fmt, ...)
{
        va_list ap;
        char buf[512];
        int n;

        if (!early_console)
                return;

        va_start(ap, fmt);
        n = vscnprintf(buf, sizeof(buf), fmt, ap);
        va_end(ap);

        early_console->write(early_console, buf, n);
}
#endif

static void set_user_specified(struct console_cmdline *c, bool user_specified)
{
        if (!user_specified)
                return;

        /*
         * @c console was defined by the user on the command line.
         * Do not clear when added twice also by SPCR or the device tree.
         */
        c->user_specified = true;
        /* At least one console defined by the user on the command line. */
        console_set_on_cmdline = 1;
}

static int __add_preferred_console(const char *name, const short idx,
                                   const char *devname, char *options,
                                   char *brl_options, bool user_specified)
{
        struct console_cmdline *c;
        int i;

        if (!name && !devname)
                return -EINVAL;

        /*
         * We use a signed short index for struct console for device drivers to
         * indicate a not yet assigned index or port. However, a negative index
         * value is not valid when the console name and index are defined on
         * the command line.
         */
        if (name && idx < 0)
                return -EINVAL;

        /*
         *        See if this tty is not yet registered, and
         *        if we have a slot free.
         */
        for (i = 0, c = console_cmdline;
             i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
             i++, c++) {
                if ((name && strcmp(c->name, name) == 0 && c->index == idx) ||
                    (devname && strcmp(c->devname, devname) == 0)) {
                        if (!brl_options)
                                preferred_console = i;
                        set_user_specified(c, user_specified);
                        return 0;
                }
        }
        if (i == MAX_CMDLINECONSOLES)
                return -E2BIG;
        if (!brl_options)
                preferred_console = i;
        if (name)
                strscpy(c->name, name);
        if (devname)
                strscpy(c->devname, devname);
        c->options = options;
        set_user_specified(c, user_specified);
        braille_set_options(c, brl_options);

        c->index = idx;
        return 0;
}

static int __init console_msg_format_setup(char *str)
{
        if (!strcmp(str, "syslog"))
                console_msg_format = MSG_FORMAT_SYSLOG;
        if (!strcmp(str, "default"))
                console_msg_format = MSG_FORMAT_DEFAULT;
        return 1;
}
__setup("console_msg_format=", console_msg_format_setup);

/*
 * Set up a console.  Called via do_early_param() in init/main.c
 * for each "console=" parameter in the boot command line.
 */
static int __init console_setup(char *str)
{
        static_assert(sizeof(console_cmdline[0].devname) >= sizeof(console_cmdline[0].name) + 4);
        char buf[sizeof(console_cmdline[0].devname)];
        char *brl_options = NULL;
        char *ttyname = NULL;
        char *devname = NULL;
        char *options;
        char *s;
        int idx;

        /*
         * console="" or console=null have been suggested as a way to
         * disable console output. Use ttynull that has been created
         * for exactly this purpose.
         */
        if (str[0] == 0 || strcmp(str, "null") == 0) {
                __add_preferred_console("ttynull", 0, NULL, NULL, NULL, true);
                return 1;
        }

        if (_braille_console_setup(&str, &brl_options))
                return 1;

        /* For a DEVNAME:0.0 style console the character device is unknown early */
        if (strchr(str, ':'))
                devname = buf;
        else
                ttyname = buf;

        /*
         * Decode str into name, index, options.
         */
        if (ttyname && isdigit(str[0]))
                scnprintf(buf, sizeof(buf), "ttyS%s", str);
        else
                strscpy(buf, str);

        options = strchr(str, ',');
        if (options)
                *(options++) = 0;

#ifdef __sparc__
        if (!strcmp(str, "ttya"))
                strscpy(buf, "ttyS0");
        if (!strcmp(str, "ttyb"))
                strscpy(buf, "ttyS1");
#endif

        for (s = buf; *s; s++)
                if ((ttyname && isdigit(*s)) || *s == ',')
                        break;

        /* @idx will get defined when devname matches. */
        if (devname)
                idx = -1;
        else
                idx = simple_strtoul(s, NULL, 10);

        *s = 0;

        __add_preferred_console(ttyname, idx, devname, options, brl_options, true);
        return 1;
}
__setup("console=", console_setup);

/**
 * add_preferred_console - add a device to the list of preferred consoles.
 * @name: device name
 * @idx: device index
 * @options: options for this console
 *
 * The last preferred console added will be used for kernel messages
 * and stdin/out/err for init.  Normally this is used by console_setup
 * above to handle user-supplied console arguments; however it can also
 * be used by arch-specific code either to override the user or more
 * commonly to provide a default console (ie from PROM variables) when
 * the user has not supplied one.
 */
int add_preferred_console(const char *name, const short idx, char *options)
{
        return __add_preferred_console(name, idx, NULL, options, NULL, false);
}

/**
 * match_devname_and_update_preferred_console - Update a preferred console
 *        when matching devname is found.
 * @devname: DEVNAME:0.0 style device name
 * @name: Name of the corresponding console driver, e.g. "ttyS"
 * @idx: Console index, e.g. port number.
 *
 * The function checks whether a device with the given @devname is
 * preferred via the console=DEVNAME:0.0 command line option.
 * It fills the missing console driver name and console index
 * so that a later register_console() call could find (match)
 * and enable this device.
 *
 * It might be used when a driver subsystem initializes particular
 * devices with already known DEVNAME:0.0 style names. And it
 * could predict which console driver name and index this device
 * would later get associated with.
 *
 * Return: 0 on success, negative error code on failure.
 */
int match_devname_and_update_preferred_console(const char *devname,
                                               const char *name,
                                               const short idx)
{
        struct console_cmdline *c = console_cmdline;
        int i;

        if (!devname || !strlen(devname) || !name || !strlen(name) || idx < 0)
                return -EINVAL;

        for (i = 0; i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
             i++, c++) {
                if (!strcmp(devname, c->devname)) {
                        pr_info("associate the preferred console \"%s\" with \"%s%d\"\n",
                                devname, name, idx);
                        strscpy(c->name, name);
                        c->index = idx;
                        return 0;
                }
        }

        return -ENOENT;
}
EXPORT_SYMBOL_GPL(match_devname_and_update_preferred_console);

bool console_suspend_enabled = true;
EXPORT_SYMBOL(console_suspend_enabled);

static int __init console_suspend_disable(char *str)
{
        console_suspend_enabled = false;
        return 1;
}
__setup("no_console_suspend", console_suspend_disable);
module_param_named(console_suspend, console_suspend_enabled,
                bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(console_suspend, "suspend console during suspend"
        " and hibernate operations");

static bool printk_console_no_auto_verbose;

void console_verbose(void)
{
        if (console_loglevel && !printk_console_no_auto_verbose)
                console_loglevel = CONSOLE_LOGLEVEL_MOTORMOUTH;
}
EXPORT_SYMBOL_GPL(console_verbose);

module_param_named(console_no_auto_verbose, printk_console_no_auto_verbose, bool, 0644);
MODULE_PARM_DESC(console_no_auto_verbose, "Disable console loglevel raise to highest on oops/panic/etc");

/**
 * console_suspend_all - suspend the console subsystem
 *
 * This disables printk() while we go into suspend states
 */
void console_suspend_all(void)
{
        struct console *con;

        if (!console_suspend_enabled)
                return;
        pr_info("Suspending console(s) (use no_console_suspend to debug)\n");
        pr_flush(1000, true);

        console_list_lock();
        for_each_console(con)
                console_srcu_write_flags(con, con->flags | CON_SUSPENDED);
        console_list_unlock();

        /*
         * Ensure that all SRCU list walks have completed. All printing
         * contexts must be able to see that they are suspended so that it
         * is guaranteed that all printing has stopped when this function
         * completes.
         */
        synchronize_srcu(&console_srcu);
}

void console_resume_all(void)
{
        struct console_flush_type ft;
        struct console *con;

        if (!console_suspend_enabled)
                return;

        console_list_lock();
        for_each_console(con)
                console_srcu_write_flags(con, con->flags & ~CON_SUSPENDED);
        console_list_unlock();

        /*
         * Ensure that all SRCU list walks have completed. All printing
         * contexts must be able to see they are no longer suspended so
         * that they are guaranteed to wake up and resume printing.
         */
        synchronize_srcu(&console_srcu);

        printk_get_console_flush_type(&ft);
        if (ft.nbcon_offload)
                nbcon_kthreads_wake();
        if (ft.legacy_offload)
                defer_console_output();

        pr_flush(1000, true);
}

/**
 * console_cpu_notify - print deferred console messages after CPU hotplug
 * @cpu: unused
 *
 * If printk() is called from a CPU that is not online yet, the messages
 * will be printed on the console only if there are CON_ANYTIME consoles.
 * This function is called when a new CPU comes online (or fails to come
 * up) or goes offline.
 */
static int console_cpu_notify(unsigned int cpu)
{
        struct console_flush_type ft;

        if (!cpuhp_tasks_frozen) {
                printk_get_console_flush_type(&ft);
                if (ft.nbcon_atomic)
                        nbcon_atomic_flush_pending();
                if (ft.legacy_direct) {
                        if (console_trylock())
                                console_unlock();
                }
        }
        return 0;
}

/**
 * console_lock - block the console subsystem from printing
 *
 * Acquires a lock which guarantees that no consoles will
 * be in or enter their write() callback.
 *
 * Can sleep, returns nothing.
 */
void console_lock(void)
{
        might_sleep();

        /* On panic, the console_lock must be left to the panic cpu. */
        while (other_cpu_in_panic())
                msleep(1000);

        down_console_sem();
        console_locked = 1;
        console_may_schedule = 1;
}
EXPORT_SYMBOL(console_lock);

/**
 * console_trylock - try to block the console subsystem from printing
 *
 * Try to acquire a lock which guarantees that no consoles will
 * be in or enter their write() callback.
 *
 * returns 1 on success, and 0 on failure to acquire the lock.
 */
int console_trylock(void)
{
        /* On panic, the console_lock must be left to the panic cpu. */
        if (other_cpu_in_panic())
                return 0;
        if (down_trylock_console_sem())
                return 0;
        console_locked = 1;
        console_may_schedule = 0;
        return 1;
}
EXPORT_SYMBOL(console_trylock);

int is_console_locked(void)
{
        return console_locked;
}
EXPORT_SYMBOL(is_console_locked);

static void __console_unlock(void)
{
        console_locked = 0;
        up_console_sem();
}

#ifdef CONFIG_PRINTK

/*
 * Prepend the message in @pmsg->pbufs->outbuf. This is achieved by shifting
 * the existing message over and inserting the scratchbuf message.
 *
 * @pmsg is the original printk message.
 * @fmt is the printf format of the message which will prepend the existing one.
 *
 * If there is not enough space in @pmsg->pbufs->outbuf, the existing
 * message text will be sufficiently truncated.
 *
 * If @pmsg->pbufs->outbuf is modified, @pmsg->outbuf_len is updated.
 */
__printf(2, 3)
static void console_prepend_message(struct printk_message *pmsg, const char *fmt, ...)
{
        struct printk_buffers *pbufs = pmsg->pbufs;
        const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
        const size_t outbuf_sz = sizeof(pbufs->outbuf);
        char *scratchbuf = &pbufs->scratchbuf[0];
        char *outbuf = &pbufs->outbuf[0];
        va_list args;
        size_t len;

        va_start(args, fmt);
        len = vscnprintf(scratchbuf, scratchbuf_sz, fmt, args);
        va_end(args);

        /*
         * Make sure outbuf is sufficiently large before prepending.
         * Keep at least the prefix when the message must be truncated.
         * It is a rather theoretical problem when someone tries to
         * use a minimalist buffer.
         */
        if (WARN_ON_ONCE(len + PRINTK_PREFIX_MAX >= outbuf_sz))
                return;

        if (pmsg->outbuf_len + len >= outbuf_sz) {
                /* Truncate the message, but keep it terminated. */
                pmsg->outbuf_len = outbuf_sz - (len + 1);
                outbuf[pmsg->outbuf_len] = 0;
        }

        memmove(outbuf + len, outbuf, pmsg->outbuf_len + 1);
        memcpy(outbuf, scratchbuf, len);
        pmsg->outbuf_len += len;
}

/*
 * Prepend the message in @pmsg->pbufs->outbuf with a "dropped message".
 * @pmsg->outbuf_len is updated appropriately.
 *
 * @pmsg is the printk message to prepend.
 *
 * @dropped is the dropped count to report in the dropped message.
 */
void console_prepend_dropped(struct printk_message *pmsg, unsigned long dropped)
{
        console_prepend_message(pmsg, "** %lu printk messages dropped **\n", dropped);
}

/*
 * Prepend the message in @pmsg->pbufs->outbuf with a "replay message".
 * @pmsg->outbuf_len is updated appropriately.
 *
 * @pmsg is the printk message to prepend.
 */
void console_prepend_replay(struct printk_message *pmsg)
{
        console_prepend_message(pmsg, "** replaying previous printk message **\n");
}

/*
 * Read and format the specified record (or a later record if the specified
 * record is not available).
 *
 * @pmsg will contain the formatted result. @pmsg->pbufs must point to a
 * struct printk_buffers.
 *
 * @seq is the record to read and format. If it is not available, the next
 * valid record is read.
 *
 * @is_extended specifies if the message should be formatted for extended
 * console output.
 *
 * @may_supress specifies if records may be skipped based on loglevel.
 *
 * Returns false if no record is available. Otherwise true and all fields
 * of @pmsg are valid. (See the documentation of struct printk_message
 * for information about the @pmsg fields.)
 */
bool printk_get_next_message(struct printk_message *pmsg, u64 seq,
                             bool is_extended, bool may_suppress)
{
        struct printk_buffers *pbufs = pmsg->pbufs;
        const size_t scratchbuf_sz = sizeof(pbufs->scratchbuf);
        const size_t outbuf_sz = sizeof(pbufs->outbuf);
        char *scratchbuf = &pbufs->scratchbuf[0];
        char *outbuf = &pbufs->outbuf[0];
        struct printk_info info;
        struct printk_record r;
        size_t len = 0;
        bool force_con;

        /*
         * Formatting extended messages requires a separate buffer, so use the
         * scratch buffer to read in the ringbuffer text.
         *
         * Formatting normal messages is done in-place, so read the ringbuffer
         * text directly into the output buffer.
         */
        if (is_extended)
                prb_rec_init_rd(&r, &info, scratchbuf, scratchbuf_sz);
        else
                prb_rec_init_rd(&r, &info, outbuf, outbuf_sz);

        if (!prb_read_valid(prb, seq, &r))
                return false;

        pmsg->seq = r.info->seq;
        pmsg->dropped = r.info->seq - seq;
        force_con = r.info->flags & LOG_FORCE_CON;

        /*
         * Skip records that are not forced to be printed on consoles and that
         * has level above the console loglevel.
         */
        if (!force_con && may_suppress && suppress_message_printing(r.info->level))
                goto out;

        if (is_extended) {
                len = info_print_ext_header(outbuf, outbuf_sz, r.info);
                len += msg_print_ext_body(outbuf + len, outbuf_sz - len,
                                          &r.text_buf[0], r.info->text_len, &r.info->dev_info);
        } else {
                len = record_print_text(&r, console_msg_format & MSG_FORMAT_SYSLOG, printk_time);
        }
out:
        pmsg->outbuf_len = len;
        return true;
}

/*
 * Legacy console printing from printk() caller context does not respect
 * raw_spinlock/spinlock nesting. For !PREEMPT_RT the lockdep warning is a
 * false positive. For PREEMPT_RT the false positive condition does not
 * occur.
 *
 * This map is used to temporarily establish LD_WAIT_SLEEP context for the
 * console write() callback when legacy printing to avoid false positive
 * lockdep complaints, thus allowing lockdep to continue to function for
 * real issues.
 */
#ifdef CONFIG_PREEMPT_RT
static inline void printk_legacy_allow_spinlock_enter(void) { }
static inline void printk_legacy_allow_spinlock_exit(void) { }
#else
static DEFINE_WAIT_OVERRIDE_MAP(printk_legacy_map, LD_WAIT_SLEEP);

static inline void printk_legacy_allow_spinlock_enter(void)
{
        lock_map_acquire_try(&printk_legacy_map);
}

static inline void printk_legacy_allow_spinlock_exit(void)
{
        lock_map_release(&printk_legacy_map);
}
#endif /* CONFIG_PREEMPT_RT */

/*
 * Used as the printk buffers for non-panic, serialized console printing.
 * This is for legacy (!CON_NBCON) as well as all boot (CON_BOOT) consoles.
 * Its usage requires the console_lock held.
 */
struct printk_buffers printk_shared_pbufs;

/*
 * Print one record for the given console. The record printed is whatever
 * record is the next available record for the given console.
 *
 * @handover will be set to true if a printk waiter has taken over the
 * console_lock, in which case the caller is no longer holding both the
 * console_lock and the SRCU read lock. Otherwise it is set to false.
 *
 * @cookie is the cookie from the SRCU read lock.
 *
 * Returns false if the given console has no next record to print, otherwise
 * true.
 *
 * Requires the console_lock and the SRCU read lock.
 */
static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
{
        bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
        char *outbuf = &printk_shared_pbufs.outbuf[0];
        struct printk_message pmsg = {
                .pbufs = &printk_shared_pbufs,
        };
        unsigned long flags;

        *handover = false;

        if (!printk_get_next_message(&pmsg, con->seq, is_extended, true))
                return false;

        con->dropped += pmsg.dropped;

        /* Skip messages of formatted length 0. */
        if (pmsg.outbuf_len == 0) {
                con->seq = pmsg.seq + 1;
                goto skip;
        }

        if (con->dropped && !is_extended) {
                console_prepend_dropped(&pmsg, con->dropped);
                con->dropped = 0;
        }

        /* Write everything out to the hardware. */

        if (force_legacy_kthread() && !panic_in_progress()) {
                /*
                 * With forced threading this function is in a task context
                 * (either legacy kthread or get_init_console_seq()). There
                 * is no need for concern about printk reentrance, handovers,
                 * or lockdep complaints.
                 */

                con->write(con, outbuf, pmsg.outbuf_len);
                con->seq = pmsg.seq + 1;
        } else {
                /*
                 * While actively printing out messages, if another printk()
                 * were to occur on another CPU, it may wait for this one to
                 * finish. This task can not be preempted if there is a
                 * waiter waiting to take over.
                 *
                 * Interrupts are disabled because the hand over to a waiter
                 * must not be interrupted until the hand over is completed
                 * (@console_waiter is cleared).
                 */
                printk_safe_enter_irqsave(flags);
                console_lock_spinning_enable();

                /* Do not trace print latency. */
                stop_critical_timings();

                printk_legacy_allow_spinlock_enter();
                con->write(con, outbuf, pmsg.outbuf_len);
                printk_legacy_allow_spinlock_exit();

                start_critical_timings();

                con->seq = pmsg.seq + 1;

                *handover = console_lock_spinning_disable_and_check(cookie);
                printk_safe_exit_irqrestore(flags);
        }
skip:
        return true;
}

#else

static bool console_emit_next_record(struct console *con, bool *handover, int cookie)
{
        *handover = false;
        return false;
}

static inline void printk_kthreads_check_locked(void) { }

#endif /* CONFIG_PRINTK */

/*
 * Print out all remaining records to all consoles.
 *
 * @do_cond_resched is set by the caller. It can be true only in schedulable
 * context.
 *
 * @next_seq is set to the sequence number after the last available record.
 * The value is valid only when this function returns true. It means that all
 * usable consoles are completely flushed.
 *
 * @handover will be set to true if a printk waiter has taken over the
 * console_lock, in which case the caller is no longer holding the
 * console_lock. Otherwise it is set to false.
 *
 * Returns true when there was at least one usable console and all messages
 * were flushed to all usable consoles. A returned false informs the caller
 * that everything was not flushed (either there were no usable consoles or
 * another context has taken over printing or it is a panic situation and this
 * is not the panic CPU). Regardless the reason, the caller should assume it
 * is not useful to immediately try again.
 *
 * Requires the console_lock.
 */
static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover)
{
        struct console_flush_type ft;
        bool any_usable = false;
        struct console *con;
        bool any_progress;
        int cookie;

        *next_seq = 0;
        *handover = false;

        do {
                any_progress = false;

                printk_get_console_flush_type(&ft);

                cookie = console_srcu_read_lock();
                for_each_console_srcu(con) {
                        short flags = console_srcu_read_flags(con);
                        u64 printk_seq;
                        bool progress;

                        /*
                         * console_flush_all() is only responsible for nbcon
                         * consoles when the nbcon consoles cannot print via
                         * their atomic or threaded flushing.
                         */
                        if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload))
                                continue;

                        if (!console_is_usable(con, flags, !do_cond_resched))
                                continue;
                        any_usable = true;

                        if (flags & CON_NBCON) {
                                progress = nbcon_legacy_emit_next_record(con, handover, cookie,
                                                                         !do_cond_resched);
                                printk_seq = nbcon_seq_read(con);
                        } else {
                                progress = console_emit_next_record(con, handover, cookie);
                                printk_seq = con->seq;
                        }

                        /*
                         * If a handover has occurred, the SRCU read lock
                         * is already released.
                         */
                        if (*handover)
                                return false;

                        /* Track the next of the highest seq flushed. */
                        if (printk_seq > *next_seq)
                                *next_seq = printk_seq;

                        if (!progress)
                                continue;
                        any_progress = true;

                        /* Allow panic_cpu to take over the consoles safely. */
                        if (other_cpu_in_panic())
                                goto abandon;

                        if (do_cond_resched)
                                cond_resched();
                }
                console_srcu_read_unlock(cookie);
        } while (any_progress);

        return any_usable;

abandon:
        console_srcu_read_unlock(cookie);
        return false;
}

static void __console_flush_and_unlock(void)
{
        bool do_cond_resched;
        bool handover;
        bool flushed;
        u64 next_seq;

        /*
         * Console drivers are called with interrupts disabled, so
         * @console_may_schedule should be cleared before; however, we may
         * end up dumping a lot of lines, for example, if called from
         * console registration path, and should invoke cond_resched()
         * between lines if allowable.  Not doing so can cause a very long
         * scheduling stall on a slow console leading to RCU stall and
         * softlockup warnings which exacerbate the issue with more
         * messages practically incapacitating the system. Therefore, create
         * a local to use for the printing loop.
         */
        do_cond_resched = console_may_schedule;

        do {
                console_may_schedule = 0;

                flushed = console_flush_all(do_cond_resched, &next_seq, &handover);
                if (!handover)
                        __console_unlock();

                /*
                 * Abort if there was a failure to flush all messages to all
                 * usable consoles. Either it is not possible to flush (in
                 * which case it would be an infinite loop of retrying) or
                 * another context has taken over printing.
                 */
                if (!flushed)
                        break;

                /*
                 * Some context may have added new records after
                 * console_flush_all() but before unlocking the console.
                 * Re-check if there is a new record to flush. If the trylock
                 * fails, another context is already handling the printing.
                 */
        } while (prb_read_valid(prb, next_seq, NULL) && console_trylock());
}

/**
 * console_unlock - unblock the legacy console subsystem from printing
 *
 * Releases the console_lock which the caller holds to block printing of
 * the legacy console subsystem.
 *
 * While the console_lock was held, console output may have been buffered
 * by printk(). If this is the case, console_unlock() emits the output on
 * legacy consoles prior to releasing the lock.
 *
 * console_unlock(); may be called from any context.
 */
void console_unlock(void)
{
        struct console_flush_type ft;

        printk_get_console_flush_type(&ft);
        if (ft.legacy_direct)
                __console_flush_and_unlock();
        else
                __console_unlock();
}
EXPORT_SYMBOL(console_unlock);

/**
 * console_conditional_schedule - yield the CPU if required
 *
 * If the console code is currently allowed to sleep, and
 * if this CPU should yield the CPU to another task, do
 * so here.
 *
 * Must be called within console_lock();.
 */
void __sched console_conditional_schedule(void)
{
        if (console_may_schedule)
                cond_resched();
}
EXPORT_SYMBOL(console_conditional_schedule);

void console_unblank(void)
{
        bool found_unblank = false;
        struct console *c;
        int cookie;

        /*
         * First check if there are any consoles implementing the unblank()
         * callback. If not, there is no reason to continue and take the
         * console lock, which in particular can be dangerous if
         * @oops_in_progress is set.
         */
        cookie = console_srcu_read_lock();
        for_each_console_srcu(c) {
                short flags = console_srcu_read_flags(c);

                if (flags & CON_SUSPENDED)
                        continue;

                if ((flags & CON_ENABLED) && c->unblank) {
                        found_unblank = true;
                        break;
                }
        }
        console_srcu_read_unlock(cookie);
        if (!found_unblank)
                return;

        /*
         * Stop console printing because the unblank() callback may
         * assume the console is not within its write() callback.
         *
         * If @oops_in_progress is set, this may be an atomic context.
         * In that case, attempt a trylock as best-effort.
         */
        if (oops_in_progress) {
                /* Semaphores are not NMI-safe. */
                if (in_nmi())
                        return;

                /*
                 * Attempting to trylock the console lock can deadlock
                 * if another CPU was stopped while modifying the
                 * semaphore. "Hope and pray" that this is not the
                 * current situation.
                 */
                if (down_trylock_console_sem() != 0)
                        return;
        } else
                console_lock();

        console_locked = 1;
        console_may_schedule = 0;

        cookie = console_srcu_read_lock();
        for_each_console_srcu(c) {
                short flags = console_srcu_read_flags(c);

                if (flags & CON_SUSPENDED)
                        continue;

                if ((flags & CON_ENABLED) && c->unblank)
                        c->unblank();
        }
        console_srcu_read_unlock(cookie);

        console_unlock();

        if (!oops_in_progress)
                pr_flush(1000, true);
}

/*
 * Rewind all consoles to the oldest available record.
 *
 * IMPORTANT: The function is safe only when called under
 *            console_lock(). It is not enforced because
 *            it is used as a best effort in panic().
 */
static void __console_rewind_all(void)
{
        struct console *c;
        short flags;
        int cookie;
        u64 seq;

        seq = prb_first_valid_seq(prb);

        cookie = console_srcu_read_lock();
        for_each_console_srcu(c) {
                flags = console_srcu_read_flags(c);

                if (flags & CON_NBCON) {
                        nbcon_seq_force(c, seq);
                } else {
                        /*
                         * This assignment is safe only when called under
                         * console_lock(). On panic, legacy consoles are
                         * only best effort.
                         */
                        c->seq = seq;
                }
        }
        console_srcu_read_unlock(cookie);
}

/**
 * console_flush_on_panic - flush console content on panic
 * @mode: flush all messages in buffer or just the pending ones
 *
 * Immediately output all pending messages no matter what.
 */
void console_flush_on_panic(enum con_flush_mode mode)
{
        struct console_flush_type ft;
        bool handover;
        u64 next_seq;

        /*
         * Ignore the console lock and flush out the messages. Attempting a
         * trylock would not be useful because:
         *
         *   - if it is contended, it must be ignored anyway
         *   - console_lock() and console_trylock() block and fail
         *     respectively in panic for non-panic CPUs
         *   - semaphores are not NMI-safe
         */

        /*
         * If another context is holding the console lock,
         * @console_may_schedule might be set. Clear it so that
         * this context does not call cond_resched() while flushing.
         */
        console_may_schedule = 0;

        if (mode == CONSOLE_REPLAY_ALL)
                __console_rewind_all();

        printk_get_console_flush_type(&ft);
        if (ft.nbcon_atomic)
                nbcon_atomic_flush_pending();

        /* Flush legacy consoles once allowed, even when dangerous. */
        if (legacy_allow_panic_sync)
                console_flush_all(false, &next_seq, &handover);
}

/*
 * Return the console tty driver structure and its associated index
 */
struct tty_driver *console_device(int *index)
{
        struct console *c;
        struct tty_driver *driver = NULL;
        int cookie;

        /*
         * Take console_lock to serialize device() callback with
         * other console operations. For example, fg_console is
         * modified under console_lock when switching vt.
         */
        console_lock();

        cookie = console_srcu_read_lock();
        for_each_console_srcu(c) {
                if (!c->device)
                        continue;
                driver = c->device(c, index);
                if (driver)
                        break;
        }
        console_srcu_read_unlock(cookie);

        console_unlock();
        return driver;
}

/*
 * Prevent further output on the passed console device so that (for example)
 * serial drivers can suspend console output before suspending a port, and can
 * re-enable output afterwards.
 */
void console_suspend(struct console *console)
{
        __pr_flush(console, 1000, true);
        console_list_lock();
        console_srcu_write_flags(console, console->flags & ~CON_ENABLED);
        console_list_unlock();

        /*
         * Ensure that all SRCU list walks have completed. All contexts must
         * be able to see that this console is disabled so that (for example)
         * the caller can suspend the port without risk of another context
         * using the port.
         */
        synchronize_srcu(&console_srcu);
}
EXPORT_SYMBOL(console_suspend);

void console_resume(struct console *console)
{
        struct console_flush_type ft;
        bool is_nbcon;

        console_list_lock();
        console_srcu_write_flags(console, console->flags | CON_ENABLED);
        is_nbcon = console->flags & CON_NBCON;
        console_list_unlock();

        /*
         * Ensure that all SRCU list walks have completed. The related
         * printing context must be able to see it is enabled so that
         * it is guaranteed to wake up and resume printing.
         */
        synchronize_srcu(&console_srcu);

        printk_get_console_flush_type(&ft);
        if (is_nbcon && ft.nbcon_offload)
                nbcon_kthread_wake(console);
        else if (ft.legacy_offload)
                defer_console_output();

        __pr_flush(console, 1000, true);
}
EXPORT_SYMBOL(console_resume);

#ifdef CONFIG_PRINTK
static int unregister_console_locked(struct console *console);

/* True when system boot is far enough to create printer threads. */
static bool printk_kthreads_ready __ro_after_init;

static struct task_struct *printk_legacy_kthread;

static bool legacy_kthread_should_wakeup(void)
{
        struct console_flush_type ft;
        struct console *con;
        bool ret = false;
        int cookie;

        if (kthread_should_stop())
                return true;

        printk_get_console_flush_type(&ft);

        cookie = console_srcu_read_lock();
        for_each_console_srcu(con) {
                short flags = console_srcu_read_flags(con);
                u64 printk_seq;

                /*
                 * The legacy printer thread is only responsible for nbcon
                 * consoles when the nbcon consoles cannot print via their
                 * atomic or threaded flushing.
                 */
                if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload))
                        continue;

                if (!console_is_usable(con, flags, false))
                        continue;

                if (flags & CON_NBCON) {
                        printk_seq = nbcon_seq_read(con);
                } else {
                        /*
                         * It is safe to read @seq because only this
                         * thread context updates @seq.
                         */
                        printk_seq = con->seq;
                }

                if (prb_read_valid(prb, printk_seq, NULL)) {
                        ret = true;
                        break;
                }
        }
        console_srcu_read_unlock(cookie);

        return ret;
}

static int legacy_kthread_func(void *unused)
{
        for (;;) {
                wait_event_interruptible(legacy_wait, legacy_kthread_should_wakeup());

                if (kthread_should_stop())
                        break;

                console_lock();
                __console_flush_and_unlock();
        }

        return 0;
}

static bool legacy_kthread_create(void)
{
        struct task_struct *kt;

        lockdep_assert_console_list_lock_held();

        kt = kthread_run(legacy_kthread_func, NULL, "pr/legacy");
        if (WARN_ON(IS_ERR(kt))) {
                pr_err("failed to start legacy printing thread\n");
                return false;
        }

        printk_legacy_kthread = kt;

        /*
         * It is important that console printing threads are scheduled
         * shortly after a printk call and with generous runtime budgets.
         */
        sched_set_normal(printk_legacy_kthread, -20);

        return true;
}

/**
 * printk_kthreads_shutdown - shutdown all threaded printers
 *
 * On system shutdown all threaded printers are stopped. This allows printk
 * to transition back to atomic printing, thus providing a robust mechanism
 * for the final shutdown/reboot messages to be output.
 */
static void printk_kthreads_shutdown(void)
{
        struct console *con;

        console_list_lock();
        if (printk_kthreads_running) {
                printk_kthreads_running = false;

                for_each_console(con) {
                        if (con->flags & CON_NBCON)
                                nbcon_kthread_stop(con);
                }

                /*
                 * The threads may have been stopped while printing a
                 * backlog. Flush any records left over.
                 */
                nbcon_atomic_flush_pending();
        }
        console_list_unlock();
}

static struct syscore_ops printk_syscore_ops = {
        .shutdown = printk_kthreads_shutdown,
};

/*
 * If appropriate, start nbcon kthreads and set @printk_kthreads_running.
 * If any kthreads fail to start, those consoles are unregistered.
 *
 * Must be called under console_list_lock().
 */
static void printk_kthreads_check_locked(void)
{
        struct hlist_node *tmp;
        struct console *con;

        lockdep_assert_console_list_lock_held();

        if (!printk_kthreads_ready)
                return;

        if (have_legacy_console || have_boot_console) {
                if (!printk_legacy_kthread &&
                    force_legacy_kthread() &&
                    !legacy_kthread_create()) {
                        /*
                         * All legacy consoles must be unregistered. If there
                         * are any nbcon consoles, they will set up their own
                         * kthread.
                         */
                        hlist_for_each_entry_safe(con, tmp, &console_list, node) {
                                if (con->flags & CON_NBCON)
                                        continue;

                                unregister_console_locked(con);
                        }
                }
        } else if (printk_legacy_kthread) {
                kthread_stop(printk_legacy_kthread);
                printk_legacy_kthread = NULL;
        }

        /*
         * Printer threads cannot be started as long as any boot console is
         * registered because there is no way to synchronize the hardware
         * registers between boot console code and regular console code.
         * It can only be known that there will be no new boot consoles when
         * an nbcon console is registered.
         */
        if (have_boot_console || !have_nbcon_console) {
                /* Clear flag in case all nbcon consoles unregistered. */
                printk_kthreads_running = false;
                return;
        }

        if (printk_kthreads_running)
                return;

        hlist_for_each_entry_safe(con, tmp, &console_list, node) {
                if (!(con->flags & CON_NBCON))
                        continue;

                if (!nbcon_kthread_create(con))
                        unregister_console_locked(con);
        }

        printk_kthreads_running = true;
}

static int __init printk_set_kthreads_ready(void)
{
        register_syscore_ops(&printk_syscore_ops);

        console_list_lock();
        printk_kthreads_ready = true;
        printk_kthreads_check_locked();
        console_list_unlock();

        return 0;
}
early_initcall(printk_set_kthreads_ready);
#endif /* CONFIG_PRINTK */

static int __read_mostly keep_bootcon;

static int __init keep_bootcon_setup(char *str)
{
        keep_bootcon = 1;
        pr_info("debug: skip boot console de-registration.\n");

        return 0;
}

early_param("keep_bootcon", keep_bootcon_setup);

static int console_call_setup(struct console *newcon, char *options)
{
        int err;

        if (!newcon->setup)
                return 0;

        /* Synchronize with possible boot console. */
        console_lock();
        err = newcon->setup(newcon, options);
        console_unlock();

        return err;
}

/*
 * This is called by register_console() to try to match
 * the newly registered console with any of the ones selected
 * by either the command line or add_preferred_console() and
 * setup/enable it.
 *
 * Care need to be taken with consoles that are statically
 * enabled such as netconsole
 */
static int try_enable_preferred_console(struct console *newcon,
                                        bool user_specified)
{
        struct console_cmdline *c;
        int i, err;

        for (i = 0, c = console_cmdline;
             i < MAX_CMDLINECONSOLES && (c->name[0] || c->devname[0]);
             i++, c++) {
                /* Console not yet initialized? */
                if (!c->name[0])
                        continue;
                if (c->user_specified != user_specified)
                        continue;
                if (!newcon->match ||
                    newcon->match(newcon, c->name, c->index, c->options) != 0) {
                        /* default matching */
                        BUILD_BUG_ON(sizeof(c->name) != sizeof(newcon->name));
                        if (strcmp(c->name, newcon->name) != 0)
                                continue;
                        if (newcon->index >= 0 &&
                            newcon->index != c->index)
                                continue;
                        if (newcon->index < 0)
                                newcon->index = c->index;

                        if (_braille_register_console(newcon, c))
                                return 0;

                        err = console_call_setup(newcon, c->options);
                        if (err)
                                return err;
                }
                newcon->flags |= CON_ENABLED;
                if (i == preferred_console)
                        newcon->flags |= CON_CONSDEV;
                return 0;
        }

        /*
         * Some consoles, such as pstore and netconsole, can be enabled even
         * without matching. Accept the pre-enabled consoles only when match()
         * and setup() had a chance to be called.
         */
        if (newcon->flags & CON_ENABLED && c->user_specified ==        user_specified)
                return 0;

        return -ENOENT;
}

/* Try to enable the console unconditionally */
static void try_enable_default_console(struct console *newcon)
{
        if (newcon->index < 0)
                newcon->index = 0;

        if (console_call_setup(newcon, NULL) != 0)
                return;

        newcon->flags |= CON_ENABLED;

        if (newcon->device)
                newcon->flags |= CON_CONSDEV;
}

/* Return the starting sequence number for a newly registered console. */
static u64 get_init_console_seq(struct console *newcon, bool bootcon_registered)
{
        struct console *con;
        bool handover;
        u64 init_seq;

        if (newcon->flags & (CON_PRINTBUFFER | CON_BOOT)) {
                /* Get a consistent copy of @syslog_seq. */
                mutex_lock(&syslog_lock);
                init_seq = syslog_seq;
                mutex_unlock(&syslog_lock);
        } else {
                /* Begin with next message added to ringbuffer. */
                init_seq = prb_next_seq(prb);

                /*
                 * If any enabled boot consoles are due to be unregistered
                 * shortly, some may not be caught up and may be the same
                 * device as @newcon. Since it is not known which boot console
                 * is the same device, flush all consoles and, if necessary,
                 * start with the message of the enabled boot console that is
                 * the furthest behind.
                 */
                if (bootcon_registered && !keep_bootcon) {
                        /*
                         * Hold the console_lock to stop console printing and
                         * guarantee safe access to console->seq.
                         */
                        console_lock();

                        /*
                         * Flush all consoles and set the console to start at
                         * the next unprinted sequence number.
                         */
                        if (!console_flush_all(true, &init_seq, &handover)) {
                                /*
                                 * Flushing failed. Just choose the lowest
                                 * sequence of the enabled boot consoles.
                                 */

                                /*
                                 * If there was a handover, this context no
                                 * longer holds the console_lock.
                                 */
                                if (handover)
                                        console_lock();

                                init_seq = prb_next_seq(prb);
                                for_each_console(con) {
                                        u64 seq;

                                        if (!(con->flags & CON_BOOT) ||
                                            !(con->flags & CON_ENABLED)) {
                                                continue;
                                        }

                                        if (con->flags & CON_NBCON)
                                                seq = nbcon_seq_read(con);
                                        else
                                                seq = con->seq;

                                        if (seq < init_seq)
                                                init_seq = seq;
                                }
                        }

                        console_unlock();
                }
        }

        return init_seq;
}

#define console_first()                                \
        hlist_entry(console_list.first, struct console, node)

static int unregister_console_locked(struct console *console);

/*
 * The console driver calls this routine during kernel initialization
 * to register the console printing procedure with printk() and to
 * print any messages that were printed by the kernel before the
 * console driver was initialized.
 *
 * This can happen pretty early during the boot process (because of
 * early_printk) - sometimes before setup_arch() completes - be careful
 * of what kernel features are used - they may not be initialised yet.
 *
 * There are two types of consoles - bootconsoles (early_printk) and
 * "real" consoles (everything which is not a bootconsole) which are
 * handled differently.
 *  - Any number of bootconsoles can be registered at any time.
 *  - As soon as a "real" console is registered, all bootconsoles
 *    will be unregistered automatically.
 *  - Once a "real" console is registered, any attempt to register a
 *    bootconsoles will be rejected
 */
void register_console(struct console *newcon)
{
        bool use_device_lock = (newcon->flags & CON_NBCON) && newcon->write_atomic;
        bool bootcon_registered = false;
        bool realcon_registered = false;
        struct console *con;
        unsigned long flags;
        u64 init_seq;
        int err;

        console_list_lock();

        for_each_console(con) {
                if (WARN(con == newcon, "console '%s%d' already registered\n",
                                         con->name, con->index)) {
                        goto unlock;
                }

                if (con->flags & CON_BOOT)
                        bootcon_registered = true;
                else
                        realcon_registered = true;
        }

        /* Do not register boot consoles when there already is a real one. */
        if ((newcon->flags & CON_BOOT) && realcon_registered) {
                pr_info("Too late to register bootconsole %s%d\n",
                        newcon->name, newcon->index);
                goto unlock;
        }

        if (newcon->flags & CON_NBCON) {
                /*
                 * Ensure the nbcon console buffers can be allocated
                 * before modifying any global data.
                 */
                if (!nbcon_alloc(newcon))
                        goto unlock;
        }

        /*
         * See if we want to enable this console driver by default.
         *
         * Nope when a console is preferred by the command line, device
         * tree, or SPCR.
         *
         * The first real console with tty binding (driver) wins. More
         * consoles might get enabled before the right one is found.
         *
         * Note that a console with tty binding will have CON_CONSDEV
         * flag set and will be first in the list.
         */
        if (preferred_console < 0) {
                if (hlist_empty(&console_list) || !console_first()->device ||
                    console_first()->flags & CON_BOOT) {
                        try_enable_default_console(newcon);
                }
        }

        /* See if this console matches one we selected on the command line */
        err = try_enable_preferred_console(newcon, true);

        /* If not, try to match against the platform default(s) */
        if (err == -ENOENT)
                err = try_enable_preferred_console(newcon, false);

        /* printk() messages are not printed to the Braille console. */
        if (err || newcon->flags & CON_BRL) {
                if (newcon->flags & CON_NBCON)
                        nbcon_free(newcon);
                goto unlock;
        }

        /*
         * If we have a bootconsole, and are switching to a real console,
         * don't print everything out again, since when the boot console, and
         * the real console are the same physical device, it's annoying to
         * see the beginning boot messages twice
         */
        if (bootcon_registered &&
            ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV)) {
                newcon->flags &= ~CON_PRINTBUFFER;
        }

        newcon->dropped = 0;
        init_seq = get_init_console_seq(newcon, bootcon_registered);

        if (newcon->flags & CON_NBCON) {
                have_nbcon_console = true;
                nbcon_seq_force(newcon, init_seq);
        } else {
                have_legacy_console = true;
                newcon->seq = init_seq;
        }

        if (newcon->flags & CON_BOOT)
                have_boot_console = true;

        /*
         * If another context is actively using the hardware of this new
         * console, it will not be aware of the nbcon synchronization. This
         * is a risk that two contexts could access the hardware
         * simultaneously if this new console is used for atomic printing
         * and the other context is still using the hardware.
         *
         * Use the driver synchronization to ensure that the hardware is not
         * in use while this new console transitions to being registered.
         */
        if (use_device_lock)
                newcon->device_lock(newcon, &flags);

        /*
         * Put this console in the list - keep the
         * preferred driver at the head of the list.
         */
        if (hlist_empty(&console_list)) {
                /* Ensure CON_CONSDEV is always set for the head. */
                newcon->flags |= CON_CONSDEV;
                hlist_add_head_rcu(&newcon->node, &console_list);

        } else if (newcon->flags & CON_CONSDEV) {
                /* Only the new head can have CON_CONSDEV set. */
                console_srcu_write_flags(console_first(), console_first()->flags & ~CON_CONSDEV);
                hlist_add_head_rcu(&newcon->node, &console_list);

        } else {
                hlist_add_behind_rcu(&newcon->node, console_list.first);
        }

        /*
         * No need to synchronize SRCU here! The caller does not rely
         * on all contexts being able to see the new console before
         * register_console() completes.
         */

        /* This new console is now registered. */
        if (use_device_lock)
                newcon->device_unlock(newcon, flags);

        console_sysfs_notify();

        /*
         * By unregistering the bootconsoles after we enable the real console
         * we get the "console xxx enabled" message on all the consoles -
         * boot consoles, real consoles, etc - this is to ensure that end
         * users know there might be something in the kernel's log buffer that
         * went to the bootconsole (that they do not see on the real console)
         */
        con_printk(KERN_INFO, newcon, "enabled\n");
        if (bootcon_registered &&
            ((newcon->flags & (CON_CONSDEV | CON_BOOT)) == CON_CONSDEV) &&
            !keep_bootcon) {
                struct hlist_node *tmp;

                hlist_for_each_entry_safe(con, tmp, &console_list, node) {
                        if (con->flags & CON_BOOT)
                                unregister_console_locked(con);
                }
        }

        /* Changed console list, may require printer threads to start/stop. */
        printk_kthreads_check_locked();
unlock:
        console_list_unlock();
}
EXPORT_SYMBOL(register_console);

/* Must be called under console_list_lock(). */
static int unregister_console_locked(struct console *console)
{
        bool use_device_lock = (console->flags & CON_NBCON) && console->write_atomic;
        bool found_legacy_con = false;
        bool found_nbcon_con = false;
        bool found_boot_con = false;
        unsigned long flags;
        struct console *c;
        int res;

        lockdep_assert_console_list_lock_held();

        con_printk(KERN_INFO, console, "disabled\n");

        res = _braille_unregister_console(console);
        if (res < 0)
                return res;
        if (res > 0)
                return 0;

        if (!console_is_registered_locked(console))
                res = -ENODEV;
        else if (console_is_usable(console, console->flags, true))
                __pr_flush(console, 1000, true);

        /* Disable it unconditionally */
        console_srcu_write_flags(console, console->flags & ~CON_ENABLED);

        if (res < 0)
                return res;

        /*
         * Use the driver synchronization to ensure that the hardware is not
         * in use while this console transitions to being unregistered.
         */
        if (use_device_lock)
                console->device_lock(console, &flags);

        hlist_del_init_rcu(&console->node);

        if (use_device_lock)
                console->device_unlock(console, flags);

        /*
         * <HISTORICAL>
         * If this isn't the last console and it has CON_CONSDEV set, we
         * need to set it on the next preferred console.
         * </HISTORICAL>
         *
         * The above makes no sense as there is no guarantee that the next
         * console has any device attached. Oh well....
         */
        if (!hlist_empty(&console_list) && console->flags & CON_CONSDEV)
                console_srcu_write_flags(console_first(), console_first()->flags | CON_CONSDEV);

        /*
         * Ensure that all SRCU list walks have completed. All contexts
         * must not be able to see this console in the list so that any
         * exit/cleanup routines can be performed safely.
         */
        synchronize_srcu(&console_srcu);

        if (console->flags & CON_NBCON)
                nbcon_free(console);

        console_sysfs_notify();

        if (console->exit)
                res = console->exit(console);

        /*
         * With this console gone, the global flags tracking registered
         * console types may have changed. Update them.
         */
        for_each_console(c) {
                if (c->flags & CON_BOOT)
                        found_boot_con = true;

                if (c->flags & CON_NBCON)
                        found_nbcon_con = true;
                else
                        found_legacy_con = true;
        }
        if (!found_boot_con)
                have_boot_console = found_boot_con;
        if (!found_legacy_con)
                have_legacy_console = found_legacy_con;
        if (!found_nbcon_con)
                have_nbcon_console = found_nbcon_con;

        /* Changed console list, may require printer threads to start/stop. */
        printk_kthreads_check_locked();

        return res;
}

int unregister_console(struct console *console)
{
        int res;

        console_list_lock();
        res = unregister_console_locked(console);
        console_list_unlock();
        return res;
}
EXPORT_SYMBOL(unregister_console);

/**
 * console_force_preferred_locked - force a registered console preferred
 * @con: The registered console to force preferred.
 *
 * Must be called under console_list_lock().
 */
void console_force_preferred_locked(struct console *con)
{
        struct console *cur_pref_con;

        if (!console_is_registered_locked(con))
                return;

        cur_pref_con = console_first();

        /* Already preferred? */
        if (cur_pref_con == con)
                return;

        /*
         * Delete, but do not re-initialize the entry. This allows the console
         * to continue to appear registered (via any hlist_unhashed_lockless()
         * checks), even though it was briefly removed from the console list.
         */
        hlist_del_rcu(&con->node);

        /*
         * Ensure that all SRCU list walks have completed so that the console
         * can be added to the beginning of the console list and its forward
         * list pointer can be re-initialized.
         */
        synchronize_srcu(&console_srcu);

        con->flags |= CON_CONSDEV;
        WARN_ON(!con->device);

        /* Only the new head can have CON_CONSDEV set. */
        console_srcu_write_flags(cur_pref_con, cur_pref_con->flags & ~CON_CONSDEV);
        hlist_add_head_rcu(&con->node, &console_list);
}
EXPORT_SYMBOL(console_force_preferred_locked);

/*
 * Initialize the console device. This is called *early*, so
 * we can't necessarily depend on lots of kernel help here.
 * Just do some early initializations, and do the complex setup
 * later.
 */
void __init console_init(void)
{
        int ret;
        initcall_t call;
        initcall_entry_t *ce;

#ifdef CONFIG_NULL_TTY_DEFAULT_CONSOLE
        if (!console_set_on_cmdline)
                add_preferred_console("ttynull", 0, NULL);
#endif

        /* Setup the default TTY line discipline. */
        n_tty_init();

        /*
         * set up the console device so that later boot sequences can
         * inform about problems etc..
         */
        ce = __con_initcall_start;
        trace_initcall_level("console");
        while (ce < __con_initcall_end) {
                call = initcall_from_entry(ce);
                trace_initcall_start(call);
                ret = call();
                trace_initcall_finish(call, ret);
                ce++;
        }
}

/*
 * Some boot consoles access data that is in the init section and which will
 * be discarded after the initcalls have been run. To make sure that no code
 * will access this data, unregister the boot consoles in a late initcall.
 *
 * If for some reason, such as deferred probe or the driver being a loadable
 * module, the real console hasn't registered yet at this point, there will
 * be a brief interval in which no messages are logged to the console, which
 * makes it difficult to diagnose problems that occur during this time.
 *
 * To mitigate this problem somewhat, only unregister consoles whose memory
 * intersects with the init section. Note that all other boot consoles will
 * get unregistered when the real preferred console is registered.
 */
static int __init printk_late_init(void)
{
        struct hlist_node *tmp;
        struct console *con;
        int ret;

        console_list_lock();
        hlist_for_each_entry_safe(con, tmp, &console_list, node) {
                if (!(con->flags & CON_BOOT))
                        continue;

                /* Check addresses that might be used for enabled consoles. */
                if (init_section_intersects(con, sizeof(*con)) ||
                    init_section_contains(con->write, 0) ||
                    init_section_contains(con->read, 0) ||
                    init_section_contains(con->device, 0) ||
                    init_section_contains(con->unblank, 0) ||
                    init_section_contains(con->data, 0)) {
                        /*
                         * Please, consider moving the reported consoles out
                         * of the init section.
                         */
                        pr_warn("bootconsole [%s%d] uses init memory and must be disabled even before the real one is ready\n",
                                con->name, con->index);
                        unregister_console_locked(con);
                }
        }
        console_list_unlock();

        ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
                                        console_cpu_notify);
        WARN_ON(ret < 0);
        ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
                                        console_cpu_notify, NULL);
        WARN_ON(ret < 0);
        printk_sysctl_init();
        return 0;
}
late_initcall(printk_late_init);

#if defined CONFIG_PRINTK
/* If @con is specified, only wait for that console. Otherwise wait for all. */
static bool __pr_flush(struct console *con, int timeout_ms, bool reset_on_progress)
{
        unsigned long timeout_jiffies = msecs_to_jiffies(timeout_ms);
        unsigned long remaining_jiffies = timeout_jiffies;
        struct console_flush_type ft;
        struct console *c;
        u64 last_diff = 0;
        u64 printk_seq;
        short flags;
        int cookie;
        u64 diff;
        u64 seq;

        /* Sorry, pr_flush() will not work this early. */
        if (system_state < SYSTEM_SCHEDULING)
                return false;

        might_sleep();

        seq = prb_next_reserve_seq(prb);

        /* Flush the consoles so that records up to @seq are printed. */
        printk_get_console_flush_type(&ft);
        if (ft.nbcon_atomic)
                nbcon_atomic_flush_pending();
        if (ft.legacy_direct) {
                console_lock();
                console_unlock();
        }

        for (;;) {
                unsigned long begin_jiffies;
                unsigned long slept_jiffies;

                diff = 0;

                /*
                 * Hold the console_lock to guarantee safe access to
                 * console->seq. Releasing console_lock flushes more
                 * records in case @seq is still not printed on all
                 * usable consoles.
                 *
                 * Holding the console_lock is not necessary if there
                 * are no legacy or boot consoles. However, such a
                 * console could register at any time. Always hold the
                 * console_lock as a precaution rather than
                 * synchronizing against register_console().
                 */
                console_lock();

                cookie = console_srcu_read_lock();
                for_each_console_srcu(c) {
                        if (con && con != c)
                                continue;

                        flags = console_srcu_read_flags(c);

                        /*
                         * If consoles are not usable, it cannot be expected
                         * that they make forward progress, so only increment
                         * @diff for usable consoles.
                         */
                        if (!console_is_usable(c, flags, true) &&
                            !console_is_usable(c, flags, false)) {
                                continue;
                        }

                        if (flags & CON_NBCON) {
                                printk_seq = nbcon_seq_read(c);
                        } else {
                                printk_seq = c->seq;
                        }

                        if (printk_seq < seq)
                                diff += seq - printk_seq;
                }
                console_srcu_read_unlock(cookie);

                if (diff != last_diff && reset_on_progress)
                        remaining_jiffies = timeout_jiffies;

                console_unlock();

                /* Note: @diff is 0 if there are no usable consoles. */
                if (diff == 0 || remaining_jiffies == 0)
                        break;

                /* msleep(1) might sleep much longer. Check time by jiffies. */
                begin_jiffies = jiffies;
                msleep(1);
                slept_jiffies = jiffies - begin_jiffies;

                remaining_jiffies -= min(slept_jiffies, remaining_jiffies);

                last_diff = diff;
        }

        return (diff == 0);
}

/**
 * pr_flush() - Wait for printing threads to catch up.
 *
 * @timeout_ms:        The maximum time (in ms) to wait.
 * @reset_on_progress: Reset the timeout if forward progress is seen.
 *
 * A value of 0 for @timeout_ms means no waiting will occur. A value of -1
 * represents infinite waiting.
 *
 * If @reset_on_progress is true, the timeout will be reset whenever any
 * printer has been seen to make some forward progress.
 *
 * Context: Process context. May sleep while acquiring console lock.
 * Return: true if all usable printers are caught up.
 */
bool pr_flush(int timeout_ms, bool reset_on_progress)
{
        return __pr_flush(NULL, timeout_ms, reset_on_progress);
}

/*
 * Delayed printk version, for scheduler-internal messages:
 */
#define PRINTK_PENDING_WAKEUP        0x01
#define PRINTK_PENDING_OUTPUT        0x02

static DEFINE_PER_CPU(int, printk_pending);

static void wake_up_klogd_work_func(struct irq_work *irq_work)
{
        int pending = this_cpu_xchg(printk_pending, 0);

        if (pending & PRINTK_PENDING_OUTPUT) {
                if (force_legacy_kthread()) {
                        if (printk_legacy_kthread)
                                wake_up_interruptible(&legacy_wait);
                } else {
                        if (console_trylock())
                                console_unlock();
                }
        }

        if (pending & PRINTK_PENDING_WAKEUP)
                wake_up_interruptible(&log_wait);
}

static DEFINE_PER_CPU(struct irq_work, wake_up_klogd_work) =
        IRQ_WORK_INIT_LAZY(wake_up_klogd_work_func);

static void __wake_up_klogd(int val)
{
        if (!printk_percpu_data_ready())
                return;

        preempt_disable();
        /*
         * Guarantee any new records can be seen by tasks preparing to wait
         * before this context checks if the wait queue is empty.
         *
         * The full memory barrier within wq_has_sleeper() pairs with the full
         * memory barrier within set_current_state() of
         * prepare_to_wait_event(), which is called after ___wait_event() adds
         * the waiter but before it has checked the wait condition.
         *
         * This pairs with devkmsg_read:A and syslog_print:A.
         */
        if (wq_has_sleeper(&log_wait) || /* LMM(__wake_up_klogd:A) */
            (val & PRINTK_PENDING_OUTPUT)) {
                this_cpu_or(printk_pending, val);
                irq_work_queue(this_cpu_ptr(&wake_up_klogd_work));
        }
        preempt_enable();
}

/**
 * wake_up_klogd - Wake kernel logging daemon
 *
 * Use this function when new records have been added to the ringbuffer
 * and the console printing of those records has already occurred or is
 * known to be handled by some other context. This function will only
 * wake the logging daemon.
 *
 * Context: Any context.
 */
void wake_up_klogd(void)
{
        __wake_up_klogd(PRINTK_PENDING_WAKEUP);
}

/**
 * defer_console_output - Wake kernel logging daemon and trigger
 *        console printing in a deferred context
 *
 * Use this function when new records have been added to the ringbuffer,
 * this context is responsible for console printing those records, but
 * the current context is not allowed to perform the console printing.
 * Trigger an irq_work context to perform the console printing. This
 * function also wakes the logging daemon.
 *
 * Context: Any context.
 */
void defer_console_output(void)
{
        /*
         * New messages may have been added directly to the ringbuffer
         * using vprintk_store(), so wake any waiters as well.
         */
        __wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
}

void printk_trigger_flush(void)
{
        defer_console_output();
}

int vprintk_deferred(const char *fmt, va_list args)
{
        return vprintk_emit(0, LOGLEVEL_SCHED, NULL, fmt, args);
}

int _printk_deferred(const char *fmt, ...)
{
        va_list args;
        int r;

        va_start(args, fmt);
        r = vprintk_deferred(fmt, args);
        va_end(args);

        return r;
}

/*
 * printk rate limiting, lifted from the networking subsystem.
 *
 * This enforces a rate limit: not more than 10 kernel messages
 * every 5s to make a denial-of-service attack impossible.
 */
DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);

int __printk_ratelimit(const char *func)
{
        return ___ratelimit(&printk_ratelimit_state, func);
}
EXPORT_SYMBOL(__printk_ratelimit);

/**
 * printk_timed_ratelimit - caller-controlled printk ratelimiting
 * @caller_jiffies: pointer to caller's state
 * @interval_msecs: minimum interval between prints
 *
 * printk_timed_ratelimit() returns true if more than @interval_msecs
 * milliseconds have elapsed since the last time printk_timed_ratelimit()
 * returned true.
 */
bool printk_timed_ratelimit(unsigned long *caller_jiffies,
                        unsigned int interval_msecs)
{
        unsigned long elapsed = jiffies - *caller_jiffies;

        if (*caller_jiffies && elapsed <= msecs_to_jiffies(interval_msecs))
                return false;

        *caller_jiffies = jiffies;
        return true;
}
EXPORT_SYMBOL(printk_timed_ratelimit);

static DEFINE_SPINLOCK(dump_list_lock);
static LIST_HEAD(dump_list);

/**
 * kmsg_dump_register - register a kernel log dumper.
 * @dumper: pointer to the kmsg_dumper structure
 *
 * Adds a kernel log dumper to the system. The dump callback in the
 * structure will be called when the kernel oopses or panics and must be
 * set. Returns zero on success and %-EINVAL or %-EBUSY otherwise.
 */
int kmsg_dump_register(struct kmsg_dumper *dumper)
{
        unsigned long flags;
        int err = -EBUSY;

        /* The dump callback needs to be set */
        if (!dumper->dump)
                return -EINVAL;

        spin_lock_irqsave(&dump_list_lock, flags);
        /* Don't allow registering multiple times */
        if (!dumper->registered) {
                dumper->registered = 1;
                list_add_tail_rcu(&dumper->list, &dump_list);
                err = 0;
        }
        spin_unlock_irqrestore(&dump_list_lock, flags);

        return err;
}
EXPORT_SYMBOL_GPL(kmsg_dump_register);

/**
 * kmsg_dump_unregister - unregister a kmsg dumper.
 * @dumper: pointer to the kmsg_dumper structure
 *
 * Removes a dump device from the system. Returns zero on success and
 * %-EINVAL otherwise.
 */
int kmsg_dump_unregister(struct kmsg_dumper *dumper)
{
        unsigned long flags;
        int err = -EINVAL;

        spin_lock_irqsave(&dump_list_lock, flags);
        if (dumper->registered) {
                dumper->registered = 0;
                list_del_rcu(&dumper->list);
                err = 0;
        }
        spin_unlock_irqrestore(&dump_list_lock, flags);
        synchronize_rcu();

        return err;
}
EXPORT_SYMBOL_GPL(kmsg_dump_unregister);

static bool always_kmsg_dump;
module_param_named(always_kmsg_dump, always_kmsg_dump, bool, S_IRUGO | S_IWUSR);

const char *kmsg_dump_reason_str(enum kmsg_dump_reason reason)
{
        switch (reason) {
        case KMSG_DUMP_PANIC:
                return "Panic";
        case KMSG_DUMP_OOPS:
                return "Oops";
        case KMSG_DUMP_EMERG:
                return "Emergency";
        case KMSG_DUMP_SHUTDOWN:
                return "Shutdown";
        default:
                return "Unknown";
        }
}
EXPORT_SYMBOL_GPL(kmsg_dump_reason_str);

/**
 * kmsg_dump_desc - dump kernel log to kernel message dumpers.
 * @reason: the reason (oops, panic etc) for dumping
 * @desc: a short string to describe what caused the panic or oops. Can be NULL
 * if no additional description is available.
 *
 * Call each of the registered dumper's dump() callback, which can
 * retrieve the kmsg records with kmsg_dump_get_line() or
 * kmsg_dump_get_buffer().
 */
void kmsg_dump_desc(enum kmsg_dump_reason reason, const char *desc)
{
        struct kmsg_dumper *dumper;
        struct kmsg_dump_detail detail = {
                .reason = reason,
                .description = desc};

        rcu_read_lock();
        list_for_each_entry_rcu(dumper, &dump_list, list) {
                enum kmsg_dump_reason max_reason = dumper->max_reason;

                /*
                 * If client has not provided a specific max_reason, default
                 * to KMSG_DUMP_OOPS, unless always_kmsg_dump was set.
                 */
                if (max_reason == KMSG_DUMP_UNDEF) {
                        max_reason = always_kmsg_dump ? KMSG_DUMP_MAX :
                                                        KMSG_DUMP_OOPS;
                }
                if (reason > max_reason)
                        continue;

                /* invoke dumper which will iterate over records */
                dumper->dump(dumper, &detail);
        }
        rcu_read_unlock();
}

/**
 * kmsg_dump_get_line - retrieve one kmsg log line
 * @iter: kmsg dump iterator
 * @syslog: include the "<4>" prefixes
 * @line: buffer to copy the line to
 * @size: maximum size of the buffer
 * @len: length of line placed into buffer
 *
 * Start at the beginning of the kmsg buffer, with the oldest kmsg
 * record, and copy one record into the provided buffer.
 *
 * Consecutive calls will return the next available record moving
 * towards the end of the buffer with the youngest messages.
 *
 * A return value of FALSE indicates that there are no more records to
 * read.
 */
bool kmsg_dump_get_line(struct kmsg_dump_iter *iter, bool syslog,
                        char *line, size_t size, size_t *len)
{
        u64 min_seq = latched_seq_read_nolock(&clear_seq);
        struct printk_info info;
        unsigned int line_count;
        struct printk_record r;
        size_t l = 0;
        bool ret = false;

        if (iter->cur_seq < min_seq)
                iter->cur_seq = min_seq;

        prb_rec_init_rd(&r, &info, line, size);

        /* Read text or count text lines? */
        if (line) {
                if (!prb_read_valid(prb, iter->cur_seq, &r))
                        goto out;
                l = record_print_text(&r, syslog, printk_time);
        } else {
                if (!prb_read_valid_info(prb, iter->cur_seq,
                                         &info, &line_count)) {
                        goto out;
                }
                l = get_record_print_text_size(&info, line_count, syslog,
                                               printk_time);

        }

        iter->cur_seq = r.info->seq + 1;
        ret = true;
out:
        if (len)
                *len = l;
        return ret;
}
EXPORT_SYMBOL_GPL(kmsg_dump_get_line);

/**
 * kmsg_dump_get_buffer - copy kmsg log lines
 * @iter: kmsg dump iterator
 * @syslog: include the "<4>" prefixes
 * @buf: buffer to copy the line to
 * @size: maximum size of the buffer
 * @len_out: length of line placed into buffer
 *
 * Start at the end of the kmsg buffer and fill the provided buffer
 * with as many of the *youngest* kmsg records that fit into it.
 * If the buffer is large enough, all available kmsg records will be
 * copied with a single call.
 *
 * Consecutive calls will fill the buffer with the next block of
 * available older records, not including the earlier retrieved ones.
 *
 * A return value of FALSE indicates that there are no more records to
 * read.
 */
bool kmsg_dump_get_buffer(struct kmsg_dump_iter *iter, bool syslog,
                          char *buf, size_t size, size_t *len_out)
{
        u64 min_seq = latched_seq_read_nolock(&clear_seq);
        struct printk_info info;
        struct printk_record r;
        u64 seq;
        u64 next_seq;
        size_t len = 0;
        bool ret = false;
        bool time = printk_time;

        if (!buf || !size)
                goto out;

        if (iter->cur_seq < min_seq)
                iter->cur_seq = min_seq;

        if (prb_read_valid_info(prb, iter->cur_seq, &info, NULL)) {
                if (info.seq != iter->cur_seq) {
                        /* messages are gone, move to first available one */
                        iter->cur_seq = info.seq;
                }
        }

        /* last entry */
        if (iter->cur_seq >= iter->next_seq)
                goto out;

        /*
         * Find first record that fits, including all following records,
         * into the user-provided buffer for this dump. Pass in size-1
         * because this function (by way of record_print_text()) will
         * not write more than size-1 bytes of text into @buf.
         */
        seq = find_first_fitting_seq(iter->cur_seq, iter->next_seq,
                                     size - 1, syslog, time);

        /*
         * Next kmsg_dump_get_buffer() invocation will dump block of
         * older records stored right before this one.
         */
        next_seq = seq;

        prb_rec_init_rd(&r, &info, buf, size);

        prb_for_each_record(seq, prb, seq, &r) {
                if (r.info->seq >= iter->next_seq)
                        break;

                len += record_print_text(&r, syslog, time);

                /* Adjust record to store to remaining buffer space. */
                prb_rec_init_rd(&r, &info, buf + len, size - len);
        }

        iter->next_seq = next_seq;
        ret = true;
out:
        if (len_out)
                *len_out = len;
        return ret;
}
EXPORT_SYMBOL_GPL(kmsg_dump_get_buffer);

/**
 * kmsg_dump_rewind - reset the iterator
 * @iter: kmsg dump iterator
 *
 * Reset the dumper's iterator so that kmsg_dump_get_line() and
 * kmsg_dump_get_buffer() can be called again and used multiple
 * times within the same dumper.dump() callback.
 */
void kmsg_dump_rewind(struct kmsg_dump_iter *iter)
{
        iter->cur_seq = latched_seq_read_nolock(&clear_seq);
        iter->next_seq = prb_next_seq(prb);
}
EXPORT_SYMBOL_GPL(kmsg_dump_rewind);

/**
 * console_try_replay_all - try to replay kernel log on consoles
 *
 * Try to obtain lock on console subsystem and replay all
 * available records in printk buffer on the consoles.
 * Does nothing if lock is not obtained.
 *
 * Context: Any, except for NMI.
 */
void console_try_replay_all(void)
{
        struct console_flush_type ft;

        printk_get_console_flush_type(&ft);
        if (console_trylock()) {
                __console_rewind_all();
                if (ft.nbcon_atomic)
                        nbcon_atomic_flush_pending();
                if (ft.nbcon_offload)
                        nbcon_kthreads_wake();
                if (ft.legacy_offload)
                        defer_console_output();
                /* Consoles are flushed as part of console_unlock(). */
                console_unlock();
        }
}
#endif

#ifdef CONFIG_SMP
static atomic_t printk_cpu_sync_owner = ATOMIC_INIT(-1);
static atomic_t printk_cpu_sync_nested = ATOMIC_INIT(0);

bool is_printk_cpu_sync_owner(void)
{
        return (atomic_read(&printk_cpu_sync_owner) == raw_smp_processor_id());
}

/**
 * __printk_cpu_sync_wait() - Busy wait until the printk cpu-reentrant
 *                            spinning lock is not owned by any CPU.
 *
 * Context: Any context.
 */
void __printk_cpu_sync_wait(void)
{
        do {
                cpu_relax();
        } while (atomic_read(&printk_cpu_sync_owner) != -1);
}
EXPORT_SYMBOL(__printk_cpu_sync_wait);

/**
 * __printk_cpu_sync_try_get() - Try to acquire the printk cpu-reentrant
 *                               spinning lock.
 *
 * If no processor has the lock, the calling processor takes the lock and
 * becomes the owner. If the calling processor is already the owner of the
 * lock, this function succeeds immediately.
 *
 * Context: Any context. Expects interrupts to be disabled.
 * Return: 1 on success, otherwise 0.
 */
int __printk_cpu_sync_try_get(void)
{
        int cpu;
        int old;

        cpu = smp_processor_id();

        /*
         * Guarantee loads and stores from this CPU when it is the lock owner
         * are _not_ visible to the previous lock owner. This pairs with
         * __printk_cpu_sync_put:B.
         *
         * Memory barrier involvement:
         *
         * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
         * then __printk_cpu_sync_put:A can never read from
         * __printk_cpu_sync_try_get:B.
         *
         * Relies on:
         *
         * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
         * of the previous CPU
         *    matching
         * ACQUIRE from __printk_cpu_sync_try_get:A to
         * __printk_cpu_sync_try_get:B of this CPU
         */
        old = atomic_cmpxchg_acquire(&printk_cpu_sync_owner, -1,
                                     cpu); /* LMM(__printk_cpu_sync_try_get:A) */
        if (old == -1) {
                /*
                 * This CPU is now the owner and begins loading/storing
                 * data: LMM(__printk_cpu_sync_try_get:B)
                 */
                return 1;

        } else if (old == cpu) {
                /* This CPU is already the owner. */
                atomic_inc(&printk_cpu_sync_nested);
                return 1;
        }

        return 0;
}
EXPORT_SYMBOL(__printk_cpu_sync_try_get);

/**
 * __printk_cpu_sync_put() - Release the printk cpu-reentrant spinning lock.
 *
 * The calling processor must be the owner of the lock.
 *
 * Context: Any context. Expects interrupts to be disabled.
 */
void __printk_cpu_sync_put(void)
{
        if (atomic_read(&printk_cpu_sync_nested)) {
                atomic_dec(&printk_cpu_sync_nested);
                return;
        }

        /*
         * This CPU is finished loading/storing data:
         * LMM(__printk_cpu_sync_put:A)
         */

        /*
         * Guarantee loads and stores from this CPU when it was the
         * lock owner are visible to the next lock owner. This pairs
         * with __printk_cpu_sync_try_get:A.
         *
         * Memory barrier involvement:
         *
         * If __printk_cpu_sync_try_get:A reads from __printk_cpu_sync_put:B,
         * then __printk_cpu_sync_try_get:B reads from __printk_cpu_sync_put:A.
         *
         * Relies on:
         *
         * RELEASE from __printk_cpu_sync_put:A to __printk_cpu_sync_put:B
         * of this CPU
         *    matching
         * ACQUIRE from __printk_cpu_sync_try_get:A to
         * __printk_cpu_sync_try_get:B of the next CPU
         */
        atomic_set_release(&printk_cpu_sync_owner,
                           -1); /* LMM(__printk_cpu_sync_put:B) */
}
EXPORT_SYMBOL(__printk_cpu_sync_put);
#endif /* CONFIG_SMP */




















































































































































































    8 
















    3 





    7 








    7 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
// SPDX-License-Identifier: GPL-2.0-only
/*
 * IRQ offload/bypass manager
 *
 * Copyright (C) 2015 Red Hat, Inc.
 * Copyright (c) 2015 Linaro Ltd.
 *
 * Various virtualization hardware acceleration techniques allow bypassing or
 * offloading interrupts received from devices around the host kernel.  Posted
 * Interrupts on Intel VT-d systems can allow interrupts to be received
 * directly by a virtual machine.  ARM IRQ Forwarding allows forwarded physical
 * interrupts to be directly deactivated by the guest.  This manager allows
 * interrupt producers and consumers to find each other to enable this sort of
 * bypass.
 */

#include <linux/irqbypass.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mutex.h>

MODULE_LICENSE("GPL v2");
MODULE_DESCRIPTION("IRQ bypass manager utility module");

static LIST_HEAD(producers);
static LIST_HEAD(consumers);
static DEFINE_MUTEX(lock);

/* @lock must be held when calling connect */
static int __connect(struct irq_bypass_producer *prod,
                     struct irq_bypass_consumer *cons)
{
        int ret = 0;

        if (prod->stop)
                prod->stop(prod);
        if (cons->stop)
                cons->stop(cons);

        if (prod->add_consumer)
                ret = prod->add_consumer(prod, cons);

        if (!ret) {
                ret = cons->add_producer(cons, prod);
                if (ret && prod->del_consumer)
                        prod->del_consumer(prod, cons);
        }

        if (cons->start)
                cons->start(cons);
        if (prod->start)
                prod->start(prod);

        return ret;
}

/* @lock must be held when calling disconnect */
static void __disconnect(struct irq_bypass_producer *prod,
                         struct irq_bypass_consumer *cons)
{
        if (prod->stop)
                prod->stop(prod);
        if (cons->stop)
                cons->stop(cons);

        cons->del_producer(cons, prod);

        if (prod->del_consumer)
                prod->del_consumer(prod, cons);

        if (cons->start)
                cons->start(cons);
        if (prod->start)
                prod->start(prod);
}

/**
 * irq_bypass_register_producer - register IRQ bypass producer
 * @producer: pointer to producer structure
 *
 * Add the provided IRQ producer to the list of producers and connect
 * with any matching token found on the IRQ consumers list.
 */
int irq_bypass_register_producer(struct irq_bypass_producer *producer)
{
        struct irq_bypass_producer *tmp;
        struct irq_bypass_consumer *consumer;
        int ret;

        if (!producer->token)
                return -EINVAL;

        might_sleep();

        if (!try_module_get(THIS_MODULE))
                return -ENODEV;

        mutex_lock(&lock);

        list_for_each_entry(tmp, &producers, node) {
                if (tmp->token == producer->token) {
                        ret = -EBUSY;
                        goto out_err;
                }
        }

        list_for_each_entry(consumer, &consumers, node) {
                if (consumer->token == producer->token) {
                        ret = __connect(producer, consumer);
                        if (ret)
                                goto out_err;
                        break;
                }
        }

        list_add(&producer->node, &producers);

        mutex_unlock(&lock);

        return 0;
out_err:
        mutex_unlock(&lock);
        module_put(THIS_MODULE);
        return ret;
}
EXPORT_SYMBOL_GPL(irq_bypass_register_producer);

/**
 * irq_bypass_unregister_producer - unregister IRQ bypass producer
 * @producer: pointer to producer structure
 *
 * Remove a previously registered IRQ producer from the list of producers
 * and disconnect it from any connected IRQ consumer.
 */
void irq_bypass_unregister_producer(struct irq_bypass_producer *producer)
{
        struct irq_bypass_producer *tmp;
        struct irq_bypass_consumer *consumer;

        if (!producer->token)
                return;

        might_sleep();

        if (!try_module_get(THIS_MODULE))
                return; /* nothing in the list anyway */

        mutex_lock(&lock);

        list_for_each_entry(tmp, &producers, node) {
                if (tmp->token != producer->token)
                        continue;

                list_for_each_entry(consumer, &consumers, node) {
                        if (consumer->token == producer->token) {
                                __disconnect(producer, consumer);
                                break;
                        }
                }

                list_del(&producer->node);
                module_put(THIS_MODULE);
                break;
        }

        mutex_unlock(&lock);

        module_put(THIS_MODULE);
}
EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer);

/**
 * irq_bypass_register_consumer - register IRQ bypass consumer
 * @consumer: pointer to consumer structure
 *
 * Add the provided IRQ consumer to the list of consumers and connect
 * with any matching token found on the IRQ producer list.
 */
int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer)
{
        struct irq_bypass_consumer *tmp;
        struct irq_bypass_producer *producer;
        int ret;

        if (!consumer->token ||
            !consumer->add_producer || !consumer->del_producer)
                return -EINVAL;

        might_sleep();

        if (!try_module_get(THIS_MODULE))
                return -ENODEV;

        mutex_lock(&lock);

        list_for_each_entry(tmp, &consumers, node) {
                if (tmp->token == consumer->token || tmp == consumer) {
                        ret = -EBUSY;
                        goto out_err;
                }
        }

        list_for_each_entry(producer, &producers, node) {
                if (producer->token == consumer->token) {
                        ret = __connect(producer, consumer);
                        if (ret)
                                goto out_err;
                        break;
                }
        }

        list_add(&consumer->node, &consumers);

        mutex_unlock(&lock);

        return 0;
out_err:
        mutex_unlock(&lock);
        module_put(THIS_MODULE);
        return ret;
}
EXPORT_SYMBOL_GPL(irq_bypass_register_consumer);

/**
 * irq_bypass_unregister_consumer - unregister IRQ bypass consumer
 * @consumer: pointer to consumer structure
 *
 * Remove a previously registered IRQ consumer from the list of consumers
 * and disconnect it from any connected IRQ producer.
 */
void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer)
{
        struct irq_bypass_consumer *tmp;
        struct irq_bypass_producer *producer;

        if (!consumer->token)
                return;

        might_sleep();

        if (!try_module_get(THIS_MODULE))
                return; /* nothing in the list anyway */

        mutex_lock(&lock);

        list_for_each_entry(tmp, &consumers, node) {
                if (tmp != consumer)
                        continue;

                list_for_each_entry(producer, &producers, node) {
                        if (producer->token == consumer->token) {
                                __disconnect(producer, consumer);
                                break;
                        }
                }

                list_del(&consumer->node);
                module_put(THIS_MODULE);
                break;
        }

        mutex_unlock(&lock);

        module_put(THIS_MODULE);
}
EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer);























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 


    6 


































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2011, Intel Corporation.
 *
 * Description: Data Center Bridging netlink interface
 * Author: Lucy Liu <lucy.liu@intel.com>
 */

#include <linux/netdevice.h>
#include <linux/netlink.h>
#include <linux/slab.h>
#include <net/netlink.h>
#include <net/rtnetlink.h>
#include <linux/dcbnl.h>
#include <net/dcbevent.h>
#include <linux/rtnetlink.h>
#include <linux/init.h>
#include <net/sock.h>

/* Data Center Bridging (DCB) is a collection of Ethernet enhancements
 * intended to allow network traffic with differing requirements
 * (highly reliable, no drops vs. best effort vs. low latency) to operate
 * and co-exist on Ethernet.  Current DCB features are:
 *
 * Enhanced Transmission Selection (aka Priority Grouping [PG]) - provides a
 *   framework for assigning bandwidth guarantees to traffic classes.
 *
 * Priority-based Flow Control (PFC) - provides a flow control mechanism which
 *   can work independently for each 802.1p priority.
 *
 * Congestion Notification - provides a mechanism for end-to-end congestion
 *   control for protocols which do not have built-in congestion management.
 *
 * More information about the emerging standards for these Ethernet features
 * can be found at: http://www.ieee802.org/1/pages/dcbridges.html
 *
 * This file implements an rtnetlink interface to allow configuration of DCB
 * features for capable devices.
 */

/**************** DCB attribute policies *************************************/

/* DCB netlink attributes policy */
static const struct nla_policy dcbnl_rtnl_policy[DCB_ATTR_MAX + 1] = {
        [DCB_ATTR_IFNAME]      = {.type = NLA_NUL_STRING, .len = IFNAMSIZ - 1},
        [DCB_ATTR_STATE]       = {.type = NLA_U8},
        [DCB_ATTR_PFC_CFG]     = {.type = NLA_NESTED},
        [DCB_ATTR_PG_CFG]      = {.type = NLA_NESTED},
        [DCB_ATTR_SET_ALL]     = {.type = NLA_U8},
        [DCB_ATTR_PERM_HWADDR] = {.type = NLA_FLAG},
        [DCB_ATTR_CAP]         = {.type = NLA_NESTED},
        [DCB_ATTR_PFC_STATE]   = {.type = NLA_U8},
        [DCB_ATTR_BCN]         = {.type = NLA_NESTED},
        [DCB_ATTR_APP]         = {.type = NLA_NESTED},
        [DCB_ATTR_IEEE]               = {.type = NLA_NESTED},
        [DCB_ATTR_DCBX]        = {.type = NLA_U8},
        [DCB_ATTR_FEATCFG]     = {.type = NLA_NESTED},
};

/* DCB priority flow control to User Priority nested attributes */
static const struct nla_policy dcbnl_pfc_up_nest[DCB_PFC_UP_ATTR_MAX + 1] = {
        [DCB_PFC_UP_ATTR_0]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_1]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_2]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_3]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_4]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_5]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_6]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_7]   = {.type = NLA_U8},
        [DCB_PFC_UP_ATTR_ALL] = {.type = NLA_FLAG},
};

/* DCB priority grouping nested attributes */
static const struct nla_policy dcbnl_pg_nest[DCB_PG_ATTR_MAX + 1] = {
        [DCB_PG_ATTR_TC_0]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_1]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_2]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_3]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_4]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_5]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_6]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_7]      = {.type = NLA_NESTED},
        [DCB_PG_ATTR_TC_ALL]    = {.type = NLA_NESTED},
        [DCB_PG_ATTR_BW_ID_0]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_1]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_2]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_3]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_4]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_5]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_6]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_7]   = {.type = NLA_U8},
        [DCB_PG_ATTR_BW_ID_ALL] = {.type = NLA_FLAG},
};

/* DCB traffic class nested attributes. */
static const struct nla_policy dcbnl_tc_param_nest[DCB_TC_ATTR_PARAM_MAX + 1] = {
        [DCB_TC_ATTR_PARAM_PGID]            = {.type = NLA_U8},
        [DCB_TC_ATTR_PARAM_UP_MAPPING]      = {.type = NLA_U8},
        [DCB_TC_ATTR_PARAM_STRICT_PRIO]     = {.type = NLA_U8},
        [DCB_TC_ATTR_PARAM_BW_PCT]          = {.type = NLA_U8},
        [DCB_TC_ATTR_PARAM_ALL]             = {.type = NLA_FLAG},
};

/* DCB capabilities nested attributes. */
static const struct nla_policy dcbnl_cap_nest[DCB_CAP_ATTR_MAX + 1] = {
        [DCB_CAP_ATTR_ALL]     = {.type = NLA_FLAG},
        [DCB_CAP_ATTR_PG]      = {.type = NLA_U8},
        [DCB_CAP_ATTR_PFC]     = {.type = NLA_U8},
        [DCB_CAP_ATTR_UP2TC]   = {.type = NLA_U8},
        [DCB_CAP_ATTR_PG_TCS]  = {.type = NLA_U8},
        [DCB_CAP_ATTR_PFC_TCS] = {.type = NLA_U8},
        [DCB_CAP_ATTR_GSP]     = {.type = NLA_U8},
        [DCB_CAP_ATTR_BCN]     = {.type = NLA_U8},
        [DCB_CAP_ATTR_DCBX]    = {.type = NLA_U8},
};

/* DCB capabilities nested attributes. */
static const struct nla_policy dcbnl_numtcs_nest[DCB_NUMTCS_ATTR_MAX + 1] = {
        [DCB_NUMTCS_ATTR_ALL]     = {.type = NLA_FLAG},
        [DCB_NUMTCS_ATTR_PG]      = {.type = NLA_U8},
        [DCB_NUMTCS_ATTR_PFC]     = {.type = NLA_U8},
};

/* DCB BCN nested attributes. */
static const struct nla_policy dcbnl_bcn_nest[DCB_BCN_ATTR_MAX + 1] = {
        [DCB_BCN_ATTR_RP_0]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_1]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_2]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_3]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_4]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_5]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_6]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_7]         = {.type = NLA_U8},
        [DCB_BCN_ATTR_RP_ALL]       = {.type = NLA_FLAG},
        [DCB_BCN_ATTR_BCNA_0]       = {.type = NLA_U32},
        [DCB_BCN_ATTR_BCNA_1]       = {.type = NLA_U32},
        [DCB_BCN_ATTR_ALPHA]        = {.type = NLA_U32},
        [DCB_BCN_ATTR_BETA]         = {.type = NLA_U32},
        [DCB_BCN_ATTR_GD]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_GI]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_TMAX]         = {.type = NLA_U32},
        [DCB_BCN_ATTR_TD]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_RMIN]         = {.type = NLA_U32},
        [DCB_BCN_ATTR_W]            = {.type = NLA_U32},
        [DCB_BCN_ATTR_RD]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_RU]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_WRTT]         = {.type = NLA_U32},
        [DCB_BCN_ATTR_RI]           = {.type = NLA_U32},
        [DCB_BCN_ATTR_C]            = {.type = NLA_U32},
        [DCB_BCN_ATTR_ALL]          = {.type = NLA_FLAG},
};

/* DCB APP nested attributes. */
static const struct nla_policy dcbnl_app_nest[DCB_APP_ATTR_MAX + 1] = {
        [DCB_APP_ATTR_IDTYPE]       = {.type = NLA_U8},
        [DCB_APP_ATTR_ID]           = {.type = NLA_U16},
        [DCB_APP_ATTR_PRIORITY]     = {.type = NLA_U8},
};

/* IEEE 802.1Qaz nested attributes. */
static const struct nla_policy dcbnl_ieee_policy[DCB_ATTR_IEEE_MAX + 1] = {
        [DCB_ATTR_IEEE_ETS]            = {.len = sizeof(struct ieee_ets)},
        [DCB_ATTR_IEEE_PFC]            = {.len = sizeof(struct ieee_pfc)},
        [DCB_ATTR_IEEE_APP_TABLE]   = {.type = NLA_NESTED},
        [DCB_ATTR_IEEE_MAXRATE]   = {.len = sizeof(struct ieee_maxrate)},
        [DCB_ATTR_IEEE_QCN]         = {.len = sizeof(struct ieee_qcn)},
        [DCB_ATTR_IEEE_QCN_STATS]   = {.len = sizeof(struct ieee_qcn_stats)},
        [DCB_ATTR_DCB_BUFFER]       = {.len = sizeof(struct dcbnl_buffer)},
        [DCB_ATTR_DCB_APP_TRUST_TABLE] = {.type = NLA_NESTED},
};

/* DCB number of traffic classes nested attributes. */
static const struct nla_policy dcbnl_featcfg_nest[DCB_FEATCFG_ATTR_MAX + 1] = {
        [DCB_FEATCFG_ATTR_ALL]      = {.type = NLA_FLAG},
        [DCB_FEATCFG_ATTR_PG]       = {.type = NLA_U8},
        [DCB_FEATCFG_ATTR_PFC]      = {.type = NLA_U8},
        [DCB_FEATCFG_ATTR_APP]      = {.type = NLA_U8},
};

static LIST_HEAD(dcb_app_list);
static LIST_HEAD(dcb_rewr_list);
static DEFINE_SPINLOCK(dcb_lock);

static enum ieee_attrs_app dcbnl_app_attr_type_get(u8 selector)
{
        switch (selector) {
        case IEEE_8021QAZ_APP_SEL_ETHERTYPE:
        case IEEE_8021QAZ_APP_SEL_STREAM:
        case IEEE_8021QAZ_APP_SEL_DGRAM:
        case IEEE_8021QAZ_APP_SEL_ANY:
        case IEEE_8021QAZ_APP_SEL_DSCP:
                return DCB_ATTR_IEEE_APP;
        case DCB_APP_SEL_PCP:
                return DCB_ATTR_DCB_APP;
        default:
                return DCB_ATTR_IEEE_APP_UNSPEC;
        }
}

static bool dcbnl_app_attr_type_validate(enum ieee_attrs_app type)
{
        switch (type) {
        case DCB_ATTR_IEEE_APP:
        case DCB_ATTR_DCB_APP:
                return true;
        default:
                return false;
        }
}

static bool dcbnl_app_selector_validate(enum ieee_attrs_app type, u8 selector)
{
        return dcbnl_app_attr_type_get(selector) == type;
}

static struct sk_buff *dcbnl_newmsg(int type, u8 cmd, u32 port, u32 seq,
                                    u32 flags, struct nlmsghdr **nlhp)
{
        struct sk_buff *skb;
        struct dcbmsg *dcb;
        struct nlmsghdr *nlh;

        skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return NULL;

        nlh = nlmsg_put(skb, port, seq, type, sizeof(*dcb), flags);
        BUG_ON(!nlh);

        dcb = nlmsg_data(nlh);
        dcb->dcb_family = AF_UNSPEC;
        dcb->cmd = cmd;
        dcb->dcb_pad = 0;

        if (nlhp)
                *nlhp = nlh;

        return skb;
}

static int dcbnl_getstate(struct net_device *netdev, struct nlmsghdr *nlh,
                          u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        /* if (!tb[DCB_ATTR_STATE] || !netdev->dcbnl_ops->getstate) */
        if (!netdev->dcbnl_ops->getstate)
                return -EOPNOTSUPP;

        return nla_put_u8(skb, DCB_ATTR_STATE,
                          netdev->dcbnl_ops->getstate(netdev));
}

static int dcbnl_getpfccfg(struct net_device *netdev, struct nlmsghdr *nlh,
                           u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_PFC_UP_ATTR_MAX + 1], *nest;
        u8 value;
        int ret;
        int i;
        int getall = 0;

        if (!tb[DCB_ATTR_PFC_CFG])
                return -EINVAL;

        if (!netdev->dcbnl_ops->getpfccfg)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_PFC_UP_ATTR_MAX,
                                          tb[DCB_ATTR_PFC_CFG],
                                          dcbnl_pfc_up_nest, NULL);
        if (ret)
                return ret;

        nest = nla_nest_start_noflag(skb, DCB_ATTR_PFC_CFG);
        if (!nest)
                return -EMSGSIZE;

        if (data[DCB_PFC_UP_ATTR_ALL])
                getall = 1;

        for (i = DCB_PFC_UP_ATTR_0; i <= DCB_PFC_UP_ATTR_7; i++) {
                if (!getall && !data[i])
                        continue;

                netdev->dcbnl_ops->getpfccfg(netdev, i - DCB_PFC_UP_ATTR_0,
                                             &value);
                ret = nla_put_u8(skb, i, value);
                if (ret) {
                        nla_nest_cancel(skb, nest);
                        return ret;
                }
        }
        nla_nest_end(skb, nest);

        return 0;
}

static int dcbnl_getperm_hwaddr(struct net_device *netdev, struct nlmsghdr *nlh,
                                u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        u8 perm_addr[MAX_ADDR_LEN];

        if (!netdev->dcbnl_ops->getpermhwaddr)
                return -EOPNOTSUPP;

        memset(perm_addr, 0, sizeof(perm_addr));
        netdev->dcbnl_ops->getpermhwaddr(netdev, perm_addr);

        return nla_put(skb, DCB_ATTR_PERM_HWADDR, sizeof(perm_addr), perm_addr);
}

static int dcbnl_getcap(struct net_device *netdev, struct nlmsghdr *nlh,
                        u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_CAP_ATTR_MAX + 1], *nest;
        u8 value;
        int ret;
        int i;
        int getall = 0;

        if (!tb[DCB_ATTR_CAP])
                return -EINVAL;

        if (!netdev->dcbnl_ops->getcap)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_CAP_ATTR_MAX,
                                          tb[DCB_ATTR_CAP], dcbnl_cap_nest,
                                          NULL);
        if (ret)
                return ret;

        nest = nla_nest_start_noflag(skb, DCB_ATTR_CAP);
        if (!nest)
                return -EMSGSIZE;

        if (data[DCB_CAP_ATTR_ALL])
                getall = 1;

        for (i = DCB_CAP_ATTR_ALL+1; i <= DCB_CAP_ATTR_MAX; i++) {
                if (!getall && !data[i])
                        continue;

                if (!netdev->dcbnl_ops->getcap(netdev, i, &value)) {
                        ret = nla_put_u8(skb, i, value);
                        if (ret) {
                                nla_nest_cancel(skb, nest);
                                return ret;
                        }
                }
        }
        nla_nest_end(skb, nest);

        return 0;
}

static int dcbnl_getnumtcs(struct net_device *netdev, struct nlmsghdr *nlh,
                           u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_NUMTCS_ATTR_MAX + 1], *nest;
        u8 value;
        int ret;
        int i;
        int getall = 0;

        if (!tb[DCB_ATTR_NUMTCS])
                return -EINVAL;

        if (!netdev->dcbnl_ops->getnumtcs)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_NUMTCS_ATTR_MAX,
                                          tb[DCB_ATTR_NUMTCS],
                                          dcbnl_numtcs_nest, NULL);
        if (ret)
                return ret;

        nest = nla_nest_start_noflag(skb, DCB_ATTR_NUMTCS);
        if (!nest)
                return -EMSGSIZE;

        if (data[DCB_NUMTCS_ATTR_ALL])
                getall = 1;

        for (i = DCB_NUMTCS_ATTR_ALL+1; i <= DCB_NUMTCS_ATTR_MAX; i++) {
                if (!getall && !data[i])
                        continue;

                ret = netdev->dcbnl_ops->getnumtcs(netdev, i, &value);
                if (!ret) {
                        ret = nla_put_u8(skb, i, value);
                        if (ret) {
                                nla_nest_cancel(skb, nest);
                                return ret;
                        }
                } else
                        return -EINVAL;
        }
        nla_nest_end(skb, nest);

        return 0;
}

static int dcbnl_setnumtcs(struct net_device *netdev, struct nlmsghdr *nlh,
                           u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_NUMTCS_ATTR_MAX + 1];
        int ret;
        u8 value;
        int i;

        if (!tb[DCB_ATTR_NUMTCS])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setnumtcs)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_NUMTCS_ATTR_MAX,
                                          tb[DCB_ATTR_NUMTCS],
                                          dcbnl_numtcs_nest, NULL);
        if (ret)
                return ret;

        for (i = DCB_NUMTCS_ATTR_ALL+1; i <= DCB_NUMTCS_ATTR_MAX; i++) {
                if (data[i] == NULL)
                        continue;

                value = nla_get_u8(data[i]);

                ret = netdev->dcbnl_ops->setnumtcs(netdev, i, value);
                if (ret)
                        break;
        }

        return nla_put_u8(skb, DCB_ATTR_NUMTCS, !!ret);
}

static int dcbnl_getpfcstate(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        if (!netdev->dcbnl_ops->getpfcstate)
                return -EOPNOTSUPP;

        return nla_put_u8(skb, DCB_ATTR_PFC_STATE,
                          netdev->dcbnl_ops->getpfcstate(netdev));
}

static int dcbnl_setpfcstate(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        u8 value;

        if (!tb[DCB_ATTR_PFC_STATE])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setpfcstate)
                return -EOPNOTSUPP;

        value = nla_get_u8(tb[DCB_ATTR_PFC_STATE]);

        netdev->dcbnl_ops->setpfcstate(netdev, value);

        return nla_put_u8(skb, DCB_ATTR_PFC_STATE, 0);
}

static int dcbnl_getapp(struct net_device *netdev, struct nlmsghdr *nlh,
                        u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *app_nest;
        struct nlattr *app_tb[DCB_APP_ATTR_MAX + 1];
        u16 id;
        u8 up, idtype;
        int ret;

        if (!tb[DCB_ATTR_APP])
                return -EINVAL;

        ret = nla_parse_nested_deprecated(app_tb, DCB_APP_ATTR_MAX,
                                          tb[DCB_ATTR_APP], dcbnl_app_nest,
                                          NULL);
        if (ret)
                return ret;

        /* all must be non-null */
        if ((!app_tb[DCB_APP_ATTR_IDTYPE]) ||
            (!app_tb[DCB_APP_ATTR_ID]))
                return -EINVAL;

        /* either by eth type or by socket number */
        idtype = nla_get_u8(app_tb[DCB_APP_ATTR_IDTYPE]);
        if ((idtype != DCB_APP_IDTYPE_ETHTYPE) &&
            (idtype != DCB_APP_IDTYPE_PORTNUM))
                return -EINVAL;

        id = nla_get_u16(app_tb[DCB_APP_ATTR_ID]);

        if (netdev->dcbnl_ops->getapp) {
                ret = netdev->dcbnl_ops->getapp(netdev, idtype, id);
                if (ret < 0)
                        return ret;
                else
                        up = ret;
        } else {
                struct dcb_app app = {
                                        .selector = idtype,
                                        .protocol = id,
                                     };
                up = dcb_getapp(netdev, &app);
        }

        app_nest = nla_nest_start_noflag(skb, DCB_ATTR_APP);
        if (!app_nest)
                return -EMSGSIZE;

        ret = nla_put_u8(skb, DCB_APP_ATTR_IDTYPE, idtype);
        if (ret)
                goto out_cancel;

        ret = nla_put_u16(skb, DCB_APP_ATTR_ID, id);
        if (ret)
                goto out_cancel;

        ret = nla_put_u8(skb, DCB_APP_ATTR_PRIORITY, up);
        if (ret)
                goto out_cancel;

        nla_nest_end(skb, app_nest);

        return 0;

out_cancel:
        nla_nest_cancel(skb, app_nest);
        return ret;
}

static int dcbnl_setapp(struct net_device *netdev, struct nlmsghdr *nlh,
                        u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        int ret;
        u16 id;
        u8 up, idtype;
        struct nlattr *app_tb[DCB_APP_ATTR_MAX + 1];

        if (!tb[DCB_ATTR_APP])
                return -EINVAL;

        ret = nla_parse_nested_deprecated(app_tb, DCB_APP_ATTR_MAX,
                                          tb[DCB_ATTR_APP], dcbnl_app_nest,
                                          NULL);
        if (ret)
                return ret;

        /* all must be non-null */
        if ((!app_tb[DCB_APP_ATTR_IDTYPE]) ||
            (!app_tb[DCB_APP_ATTR_ID]) ||
            (!app_tb[DCB_APP_ATTR_PRIORITY]))
                return -EINVAL;

        /* either by eth type or by socket number */
        idtype = nla_get_u8(app_tb[DCB_APP_ATTR_IDTYPE]);
        if ((idtype != DCB_APP_IDTYPE_ETHTYPE) &&
            (idtype != DCB_APP_IDTYPE_PORTNUM))
                return -EINVAL;

        id = nla_get_u16(app_tb[DCB_APP_ATTR_ID]);
        up = nla_get_u8(app_tb[DCB_APP_ATTR_PRIORITY]);

        if (netdev->dcbnl_ops->setapp) {
                ret = netdev->dcbnl_ops->setapp(netdev, idtype, id, up);
                if (ret < 0)
                        return ret;
        } else {
                struct dcb_app app;
                app.selector = idtype;
                app.protocol = id;
                app.priority = up;
                ret = dcb_setapp(netdev, &app);
        }

        ret = nla_put_u8(skb, DCB_ATTR_APP, ret);
        dcbnl_cee_notify(netdev, RTM_SETDCB, DCB_CMD_SAPP, seq, 0);

        return ret;
}

static int __dcbnl_pg_getcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             struct nlattr **tb, struct sk_buff *skb, int dir)
{
        struct nlattr *pg_nest, *param_nest, *data;
        struct nlattr *pg_tb[DCB_PG_ATTR_MAX + 1];
        struct nlattr *param_tb[DCB_TC_ATTR_PARAM_MAX + 1];
        u8 prio, pgid, tc_pct, up_map;
        int ret;
        int getall = 0;
        int i;

        if (!tb[DCB_ATTR_PG_CFG])
                return -EINVAL;

        if (!netdev->dcbnl_ops->getpgtccfgtx ||
            !netdev->dcbnl_ops->getpgtccfgrx ||
            !netdev->dcbnl_ops->getpgbwgcfgtx ||
            !netdev->dcbnl_ops->getpgbwgcfgrx)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(pg_tb, DCB_PG_ATTR_MAX,
                                          tb[DCB_ATTR_PG_CFG], dcbnl_pg_nest,
                                          NULL);
        if (ret)
                return ret;

        pg_nest = nla_nest_start_noflag(skb, DCB_ATTR_PG_CFG);
        if (!pg_nest)
                return -EMSGSIZE;

        if (pg_tb[DCB_PG_ATTR_TC_ALL])
                getall = 1;

        for (i = DCB_PG_ATTR_TC_0; i <= DCB_PG_ATTR_TC_7; i++) {
                if (!getall && !pg_tb[i])
                        continue;

                if (pg_tb[DCB_PG_ATTR_TC_ALL])
                        data = pg_tb[DCB_PG_ATTR_TC_ALL];
                else
                        data = pg_tb[i];
                ret = nla_parse_nested_deprecated(param_tb,
                                                  DCB_TC_ATTR_PARAM_MAX, data,
                                                  dcbnl_tc_param_nest, NULL);
                if (ret)
                        goto err_pg;

                param_nest = nla_nest_start_noflag(skb, i);
                if (!param_nest)
                        goto err_pg;

                pgid = DCB_ATTR_VALUE_UNDEFINED;
                prio = DCB_ATTR_VALUE_UNDEFINED;
                tc_pct = DCB_ATTR_VALUE_UNDEFINED;
                up_map = DCB_ATTR_VALUE_UNDEFINED;

                if (dir) {
                        /* Rx */
                        netdev->dcbnl_ops->getpgtccfgrx(netdev,
                                                i - DCB_PG_ATTR_TC_0, &prio,
                                                &pgid, &tc_pct, &up_map);
                } else {
                        /* Tx */
                        netdev->dcbnl_ops->getpgtccfgtx(netdev,
                                                i - DCB_PG_ATTR_TC_0, &prio,
                                                &pgid, &tc_pct, &up_map);
                }

                if (param_tb[DCB_TC_ATTR_PARAM_PGID] ||
                    param_tb[DCB_TC_ATTR_PARAM_ALL]) {
                        ret = nla_put_u8(skb,
                                         DCB_TC_ATTR_PARAM_PGID, pgid);
                        if (ret)
                                goto err_param;
                }
                if (param_tb[DCB_TC_ATTR_PARAM_UP_MAPPING] ||
                    param_tb[DCB_TC_ATTR_PARAM_ALL]) {
                        ret = nla_put_u8(skb,
                                         DCB_TC_ATTR_PARAM_UP_MAPPING, up_map);
                        if (ret)
                                goto err_param;
                }
                if (param_tb[DCB_TC_ATTR_PARAM_STRICT_PRIO] ||
                    param_tb[DCB_TC_ATTR_PARAM_ALL]) {
                        ret = nla_put_u8(skb,
                                         DCB_TC_ATTR_PARAM_STRICT_PRIO, prio);
                        if (ret)
                                goto err_param;
                }
                if (param_tb[DCB_TC_ATTR_PARAM_BW_PCT] ||
                    param_tb[DCB_TC_ATTR_PARAM_ALL]) {
                        ret = nla_put_u8(skb, DCB_TC_ATTR_PARAM_BW_PCT,
                                         tc_pct);
                        if (ret)
                                goto err_param;
                }
                nla_nest_end(skb, param_nest);
        }

        if (pg_tb[DCB_PG_ATTR_BW_ID_ALL])
                getall = 1;
        else
                getall = 0;

        for (i = DCB_PG_ATTR_BW_ID_0; i <= DCB_PG_ATTR_BW_ID_7; i++) {
                if (!getall && !pg_tb[i])
                        continue;

                tc_pct = DCB_ATTR_VALUE_UNDEFINED;

                if (dir) {
                        /* Rx */
                        netdev->dcbnl_ops->getpgbwgcfgrx(netdev,
                                        i - DCB_PG_ATTR_BW_ID_0, &tc_pct);
                } else {
                        /* Tx */
                        netdev->dcbnl_ops->getpgbwgcfgtx(netdev,
                                        i - DCB_PG_ATTR_BW_ID_0, &tc_pct);
                }
                ret = nla_put_u8(skb, i, tc_pct);
                if (ret)
                        goto err_pg;
        }

        nla_nest_end(skb, pg_nest);

        return 0;

err_param:
        nla_nest_cancel(skb, param_nest);
err_pg:
        nla_nest_cancel(skb, pg_nest);

        return -EMSGSIZE;
}

static int dcbnl_pgtx_getcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        return __dcbnl_pg_getcfg(netdev, nlh, tb, skb, 0);
}

static int dcbnl_pgrx_getcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        return __dcbnl_pg_getcfg(netdev, nlh, tb, skb, 1);
}

static int dcbnl_setstate(struct net_device *netdev, struct nlmsghdr *nlh,
                          u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        u8 value;

        if (!tb[DCB_ATTR_STATE])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setstate)
                return -EOPNOTSUPP;

        value = nla_get_u8(tb[DCB_ATTR_STATE]);

        return nla_put_u8(skb, DCB_ATTR_STATE,
                          netdev->dcbnl_ops->setstate(netdev, value));
}

static int dcbnl_setpfccfg(struct net_device *netdev, struct nlmsghdr *nlh,
                           u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_PFC_UP_ATTR_MAX + 1];
        int i;
        int ret;
        u8 value;

        if (!tb[DCB_ATTR_PFC_CFG])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setpfccfg)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_PFC_UP_ATTR_MAX,
                                          tb[DCB_ATTR_PFC_CFG],
                                          dcbnl_pfc_up_nest, NULL);
        if (ret)
                return ret;

        for (i = DCB_PFC_UP_ATTR_0; i <= DCB_PFC_UP_ATTR_7; i++) {
                if (data[i] == NULL)
                        continue;
                value = nla_get_u8(data[i]);
                netdev->dcbnl_ops->setpfccfg(netdev,
                        data[i]->nla_type - DCB_PFC_UP_ATTR_0, value);
        }

        return nla_put_u8(skb, DCB_ATTR_PFC_CFG, 0);
}

static int dcbnl_setall(struct net_device *netdev, struct nlmsghdr *nlh,
                        u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        int ret;

        if (!tb[DCB_ATTR_SET_ALL])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setall)
                return -EOPNOTSUPP;

        ret = nla_put_u8(skb, DCB_ATTR_SET_ALL,
                         netdev->dcbnl_ops->setall(netdev));
        dcbnl_cee_notify(netdev, RTM_SETDCB, DCB_CMD_SET_ALL, seq, 0);

        return ret;
}

static int __dcbnl_pg_setcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb,
                             int dir)
{
        struct nlattr *pg_tb[DCB_PG_ATTR_MAX + 1];
        struct nlattr *param_tb[DCB_TC_ATTR_PARAM_MAX + 1];
        int ret;
        int i;
        u8 pgid;
        u8 up_map;
        u8 prio;
        u8 tc_pct;

        if (!tb[DCB_ATTR_PG_CFG])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setpgtccfgtx ||
            !netdev->dcbnl_ops->setpgtccfgrx ||
            !netdev->dcbnl_ops->setpgbwgcfgtx ||
            !netdev->dcbnl_ops->setpgbwgcfgrx)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(pg_tb, DCB_PG_ATTR_MAX,
                                          tb[DCB_ATTR_PG_CFG], dcbnl_pg_nest,
                                          NULL);
        if (ret)
                return ret;

        for (i = DCB_PG_ATTR_TC_0; i <= DCB_PG_ATTR_TC_7; i++) {
                if (!pg_tb[i])
                        continue;

                ret = nla_parse_nested_deprecated(param_tb,
                                                  DCB_TC_ATTR_PARAM_MAX,
                                                  pg_tb[i],
                                                  dcbnl_tc_param_nest, NULL);
                if (ret)
                        return ret;

                pgid = DCB_ATTR_VALUE_UNDEFINED;
                prio = DCB_ATTR_VALUE_UNDEFINED;
                tc_pct = DCB_ATTR_VALUE_UNDEFINED;
                up_map = DCB_ATTR_VALUE_UNDEFINED;

                if (param_tb[DCB_TC_ATTR_PARAM_STRICT_PRIO])
                        prio =
                            nla_get_u8(param_tb[DCB_TC_ATTR_PARAM_STRICT_PRIO]);

                if (param_tb[DCB_TC_ATTR_PARAM_PGID])
                        pgid = nla_get_u8(param_tb[DCB_TC_ATTR_PARAM_PGID]);

                if (param_tb[DCB_TC_ATTR_PARAM_BW_PCT])
                        tc_pct = nla_get_u8(param_tb[DCB_TC_ATTR_PARAM_BW_PCT]);

                if (param_tb[DCB_TC_ATTR_PARAM_UP_MAPPING])
                        up_map =
                             nla_get_u8(param_tb[DCB_TC_ATTR_PARAM_UP_MAPPING]);

                /* dir: Tx = 0, Rx = 1 */
                if (dir) {
                        /* Rx */
                        netdev->dcbnl_ops->setpgtccfgrx(netdev,
                                i - DCB_PG_ATTR_TC_0,
                                prio, pgid, tc_pct, up_map);
                } else {
                        /* Tx */
                        netdev->dcbnl_ops->setpgtccfgtx(netdev,
                                i - DCB_PG_ATTR_TC_0,
                                prio, pgid, tc_pct, up_map);
                }
        }

        for (i = DCB_PG_ATTR_BW_ID_0; i <= DCB_PG_ATTR_BW_ID_7; i++) {
                if (!pg_tb[i])
                        continue;

                tc_pct = nla_get_u8(pg_tb[i]);

                /* dir: Tx = 0, Rx = 1 */
                if (dir) {
                        /* Rx */
                        netdev->dcbnl_ops->setpgbwgcfgrx(netdev,
                                         i - DCB_PG_ATTR_BW_ID_0, tc_pct);
                } else {
                        /* Tx */
                        netdev->dcbnl_ops->setpgbwgcfgtx(netdev,
                                         i - DCB_PG_ATTR_BW_ID_0, tc_pct);
                }
        }

        return nla_put_u8(skb, DCB_ATTR_PG_CFG, 0);
}

static int dcbnl_pgtx_setcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        return __dcbnl_pg_setcfg(netdev, nlh, seq, tb, skb, 0);
}

static int dcbnl_pgrx_setcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                             u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        return __dcbnl_pg_setcfg(netdev, nlh, seq, tb, skb, 1);
}

static int dcbnl_bcn_getcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                            u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *bcn_nest;
        struct nlattr *bcn_tb[DCB_BCN_ATTR_MAX + 1];
        u8 value_byte;
        u32 value_integer;
        int ret;
        bool getall = false;
        int i;

        if (!tb[DCB_ATTR_BCN])
                return -EINVAL;

        if (!netdev->dcbnl_ops->getbcnrp ||
            !netdev->dcbnl_ops->getbcncfg)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(bcn_tb, DCB_BCN_ATTR_MAX,
                                          tb[DCB_ATTR_BCN], dcbnl_bcn_nest,
                                          NULL);
        if (ret)
                return ret;

        bcn_nest = nla_nest_start_noflag(skb, DCB_ATTR_BCN);
        if (!bcn_nest)
                return -EMSGSIZE;

        if (bcn_tb[DCB_BCN_ATTR_ALL])
                getall = true;

        for (i = DCB_BCN_ATTR_RP_0; i <= DCB_BCN_ATTR_RP_7; i++) {
                if (!getall && !bcn_tb[i])
                        continue;

                netdev->dcbnl_ops->getbcnrp(netdev, i - DCB_BCN_ATTR_RP_0,
                                            &value_byte);
                ret = nla_put_u8(skb, i, value_byte);
                if (ret)
                        goto err_bcn;
        }

        for (i = DCB_BCN_ATTR_BCNA_0; i <= DCB_BCN_ATTR_RI; i++) {
                if (!getall && !bcn_tb[i])
                        continue;

                netdev->dcbnl_ops->getbcncfg(netdev, i,
                                             &value_integer);
                ret = nla_put_u32(skb, i, value_integer);
                if (ret)
                        goto err_bcn;
        }

        nla_nest_end(skb, bcn_nest);

        return 0;

err_bcn:
        nla_nest_cancel(skb, bcn_nest);
        return ret;
}

static int dcbnl_bcn_setcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                            u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_BCN_ATTR_MAX + 1];
        int i;
        int ret;
        u8 value_byte;
        u32 value_int;

        if (!tb[DCB_ATTR_BCN])
                return -EINVAL;

        if (!netdev->dcbnl_ops->setbcncfg ||
            !netdev->dcbnl_ops->setbcnrp)
                return -EOPNOTSUPP;

        ret = nla_parse_nested_deprecated(data, DCB_BCN_ATTR_MAX,
                                          tb[DCB_ATTR_BCN], dcbnl_bcn_nest,
                                          NULL);
        if (ret)
                return ret;

        for (i = DCB_BCN_ATTR_RP_0; i <= DCB_BCN_ATTR_RP_7; i++) {
                if (data[i] == NULL)
                        continue;
                value_byte = nla_get_u8(data[i]);
                netdev->dcbnl_ops->setbcnrp(netdev,
                        data[i]->nla_type - DCB_BCN_ATTR_RP_0, value_byte);
        }

        for (i = DCB_BCN_ATTR_BCNA_0; i <= DCB_BCN_ATTR_RI; i++) {
                if (data[i] == NULL)
                        continue;
                value_int = nla_get_u32(data[i]);
                netdev->dcbnl_ops->setbcncfg(netdev,
                                             i, value_int);
        }

        return nla_put_u8(skb, DCB_ATTR_BCN, 0);
}

static int dcbnl_build_peer_app(struct net_device *netdev, struct sk_buff* skb,
                                int app_nested_type, int app_info_type,
                                int app_entry_type)
{
        struct dcb_peer_app_info info;
        struct dcb_app *table = NULL;
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        u16 app_count;
        int err;


        /**
         * retrieve the peer app configuration form the driver. If the driver
         * handlers fail exit without doing anything
         */
        err = ops->peer_getappinfo(netdev, &info, &app_count);
        if (!err && app_count) {
                table = kmalloc_array(app_count, sizeof(struct dcb_app),
                                      GFP_KERNEL);
                if (!table)
                        return -ENOMEM;

                err = ops->peer_getapptable(netdev, table);
        }

        if (!err) {
                u16 i;
                struct nlattr *app;

                /**
                 * build the message, from here on the only possible failure
                 * is due to the skb size
                 */
                err = -EMSGSIZE;

                app = nla_nest_start_noflag(skb, app_nested_type);
                if (!app)
                        goto nla_put_failure;

                if (app_info_type &&
                    nla_put(skb, app_info_type, sizeof(info), &info))
                        goto nla_put_failure;

                for (i = 0; i < app_count; i++) {
                        if (nla_put(skb, app_entry_type, sizeof(struct dcb_app),
                                    &table[i]))
                                goto nla_put_failure;
                }
                nla_nest_end(skb, app);
        }
        err = 0;

nla_put_failure:
        kfree(table);
        return err;
}

static int dcbnl_getapptrust(struct net_device *netdev, struct sk_buff *skb)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        enum ieee_attrs_app type;
        struct nlattr *apptrust;
        int nselectors, err, i;
        u8 *selectors;

        selectors = kzalloc(IEEE_8021QAZ_APP_SEL_MAX + 1, GFP_KERNEL);
        if (!selectors)
                return -ENOMEM;

        err = ops->dcbnl_getapptrust(netdev, selectors, &nselectors);
        if (err) {
                err = 0;
                goto out;
        }

        apptrust = nla_nest_start(skb, DCB_ATTR_DCB_APP_TRUST_TABLE);
        if (!apptrust) {
                err = -EMSGSIZE;
                goto out;
        }

        for (i = 0; i < nselectors; i++) {
                type = dcbnl_app_attr_type_get(selectors[i]);
                err = nla_put_u8(skb, type, selectors[i]);
                if (err) {
                        nla_nest_cancel(skb, apptrust);
                        goto out;
                }
        }
        nla_nest_end(skb, apptrust);

out:
        kfree(selectors);
        return err;
}

/* Set or delete APP table or rewrite table entries. The APP struct is validated
 * and the appropriate callback function is called.
 */
static int dcbnl_app_table_setdel(struct nlattr *attr,
                                  struct net_device *netdev,
                                  int (*setdel)(struct net_device *dev,
                                                struct dcb_app *app))
{
        struct dcb_app *app_data;
        enum ieee_attrs_app type;
        struct nlattr *attr_itr;
        int rem, err;

        nla_for_each_nested(attr_itr, attr, rem) {
                type = nla_type(attr_itr);

                if (!dcbnl_app_attr_type_validate(type))
                        continue;

                if (nla_len(attr_itr) < sizeof(struct dcb_app))
                        return -ERANGE;

                app_data = nla_data(attr_itr);

                if (!dcbnl_app_selector_validate(type, app_data->selector))
                        return -EINVAL;

                err = setdel(netdev, app_data);
                if (err)
                        return err;
        }

        return 0;
}

/* Handle IEEE 802.1Qaz/802.1Qau/802.1Qbb GET commands. */
static int dcbnl_ieee_fill(struct sk_buff *skb, struct net_device *netdev)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        struct nlattr *ieee, *app, *rewr;
        struct dcb_app_type *itr;
        int dcbx;
        int err;

        if (nla_put_string(skb, DCB_ATTR_IFNAME, netdev->name))
                return -EMSGSIZE;

        ieee = nla_nest_start_noflag(skb, DCB_ATTR_IEEE);
        if (!ieee)
                return -EMSGSIZE;

        if (ops->ieee_getets) {
                struct ieee_ets ets;
                memset(&ets, 0, sizeof(ets));
                err = ops->ieee_getets(netdev, &ets);
                if (!err &&
                    nla_put(skb, DCB_ATTR_IEEE_ETS, sizeof(ets), &ets))
                        return -EMSGSIZE;
        }

        if (ops->ieee_getmaxrate) {
                struct ieee_maxrate maxrate;
                memset(&maxrate, 0, sizeof(maxrate));
                err = ops->ieee_getmaxrate(netdev, &maxrate);
                if (!err) {
                        err = nla_put(skb, DCB_ATTR_IEEE_MAXRATE,
                                      sizeof(maxrate), &maxrate);
                        if (err)
                                return -EMSGSIZE;
                }
        }

        if (ops->ieee_getqcn) {
                struct ieee_qcn qcn;

                memset(&qcn, 0, sizeof(qcn));
                err = ops->ieee_getqcn(netdev, &qcn);
                if (!err) {
                        err = nla_put(skb, DCB_ATTR_IEEE_QCN,
                                      sizeof(qcn), &qcn);
                        if (err)
                                return -EMSGSIZE;
                }
        }

        if (ops->ieee_getqcnstats) {
                struct ieee_qcn_stats qcn_stats;

                memset(&qcn_stats, 0, sizeof(qcn_stats));
                err = ops->ieee_getqcnstats(netdev, &qcn_stats);
                if (!err) {
                        err = nla_put(skb, DCB_ATTR_IEEE_QCN_STATS,
                                      sizeof(qcn_stats), &qcn_stats);
                        if (err)
                                return -EMSGSIZE;
                }
        }

        if (ops->ieee_getpfc) {
                struct ieee_pfc pfc;
                memset(&pfc, 0, sizeof(pfc));
                err = ops->ieee_getpfc(netdev, &pfc);
                if (!err &&
                    nla_put(skb, DCB_ATTR_IEEE_PFC, sizeof(pfc), &pfc))
                        return -EMSGSIZE;
        }

        if (ops->dcbnl_getbuffer) {
                struct dcbnl_buffer buffer;

                memset(&buffer, 0, sizeof(buffer));
                err = ops->dcbnl_getbuffer(netdev, &buffer);
                if (!err &&
                    nla_put(skb, DCB_ATTR_DCB_BUFFER, sizeof(buffer), &buffer))
                        return -EMSGSIZE;
        }

        app = nla_nest_start_noflag(skb, DCB_ATTR_IEEE_APP_TABLE);
        if (!app)
                return -EMSGSIZE;

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->ifindex == netdev->ifindex) {
                        enum ieee_attrs_app type =
                                dcbnl_app_attr_type_get(itr->app.selector);
                        err = nla_put(skb, type, sizeof(itr->app), &itr->app);
                        if (err) {
                                spin_unlock_bh(&dcb_lock);
                                return -EMSGSIZE;
                        }
                }
        }

        if (netdev->dcbnl_ops->getdcbx)
                dcbx = netdev->dcbnl_ops->getdcbx(netdev);
        else
                dcbx = -EOPNOTSUPP;

        spin_unlock_bh(&dcb_lock);
        nla_nest_end(skb, app);

        rewr = nla_nest_start(skb, DCB_ATTR_DCB_REWR_TABLE);
        if (!rewr)
                return -EMSGSIZE;

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_rewr_list, list) {
                if (itr->ifindex == netdev->ifindex) {
                        enum ieee_attrs_app type =
                                dcbnl_app_attr_type_get(itr->app.selector);
                        err = nla_put(skb, type, sizeof(itr->app), &itr->app);
                        if (err) {
                                spin_unlock_bh(&dcb_lock);
                                nla_nest_cancel(skb, rewr);
                                return -EMSGSIZE;
                        }
                }
        }

        spin_unlock_bh(&dcb_lock);
        nla_nest_end(skb, rewr);

        if (ops->dcbnl_getapptrust) {
                err = dcbnl_getapptrust(netdev, skb);
                if (err)
                        return err;
        }

        /* get peer info if available */
        if (ops->ieee_peer_getets) {
                struct ieee_ets ets;
                memset(&ets, 0, sizeof(ets));
                err = ops->ieee_peer_getets(netdev, &ets);
                if (!err &&
                    nla_put(skb, DCB_ATTR_IEEE_PEER_ETS, sizeof(ets), &ets))
                        return -EMSGSIZE;
        }

        if (ops->ieee_peer_getpfc) {
                struct ieee_pfc pfc;
                memset(&pfc, 0, sizeof(pfc));
                err = ops->ieee_peer_getpfc(netdev, &pfc);
                if (!err &&
                    nla_put(skb, DCB_ATTR_IEEE_PEER_PFC, sizeof(pfc), &pfc))
                        return -EMSGSIZE;
        }

        if (ops->peer_getappinfo && ops->peer_getapptable) {
                err = dcbnl_build_peer_app(netdev, skb,
                                           DCB_ATTR_IEEE_PEER_APP,
                                           DCB_ATTR_IEEE_APP_UNSPEC,
                                           DCB_ATTR_IEEE_APP);
                if (err)
                        return -EMSGSIZE;
        }

        nla_nest_end(skb, ieee);
        if (dcbx >= 0) {
                err = nla_put_u8(skb, DCB_ATTR_DCBX, dcbx);
                if (err)
                        return -EMSGSIZE;
        }

        return 0;
}

static int dcbnl_cee_pg_fill(struct sk_buff *skb, struct net_device *dev,
                             int dir)
{
        u8 pgid, up_map, prio, tc_pct;
        const struct dcbnl_rtnl_ops *ops = dev->dcbnl_ops;
        int i = dir ? DCB_ATTR_CEE_TX_PG : DCB_ATTR_CEE_RX_PG;
        struct nlattr *pg = nla_nest_start_noflag(skb, i);

        if (!pg)
                return -EMSGSIZE;

        for (i = DCB_PG_ATTR_TC_0; i <= DCB_PG_ATTR_TC_7; i++) {
                struct nlattr *tc_nest = nla_nest_start_noflag(skb, i);

                if (!tc_nest)
                        return -EMSGSIZE;

                pgid = DCB_ATTR_VALUE_UNDEFINED;
                prio = DCB_ATTR_VALUE_UNDEFINED;
                tc_pct = DCB_ATTR_VALUE_UNDEFINED;
                up_map = DCB_ATTR_VALUE_UNDEFINED;

                if (!dir)
                        ops->getpgtccfgrx(dev, i - DCB_PG_ATTR_TC_0,
                                          &prio, &pgid, &tc_pct, &up_map);
                else
                        ops->getpgtccfgtx(dev, i - DCB_PG_ATTR_TC_0,
                                          &prio, &pgid, &tc_pct, &up_map);

                if (nla_put_u8(skb, DCB_TC_ATTR_PARAM_PGID, pgid) ||
                    nla_put_u8(skb, DCB_TC_ATTR_PARAM_UP_MAPPING, up_map) ||
                    nla_put_u8(skb, DCB_TC_ATTR_PARAM_STRICT_PRIO, prio) ||
                    nla_put_u8(skb, DCB_TC_ATTR_PARAM_BW_PCT, tc_pct))
                        return -EMSGSIZE;
                nla_nest_end(skb, tc_nest);
        }

        for (i = DCB_PG_ATTR_BW_ID_0; i <= DCB_PG_ATTR_BW_ID_7; i++) {
                tc_pct = DCB_ATTR_VALUE_UNDEFINED;

                if (!dir)
                        ops->getpgbwgcfgrx(dev, i - DCB_PG_ATTR_BW_ID_0,
                                           &tc_pct);
                else
                        ops->getpgbwgcfgtx(dev, i - DCB_PG_ATTR_BW_ID_0,
                                           &tc_pct);
                if (nla_put_u8(skb, i, tc_pct))
                        return -EMSGSIZE;
        }
        nla_nest_end(skb, pg);
        return 0;
}

static int dcbnl_cee_fill(struct sk_buff *skb, struct net_device *netdev)
{
        struct nlattr *cee, *app;
        struct dcb_app_type *itr;
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        int dcbx, i, err = -EMSGSIZE;
        u8 value;

        if (nla_put_string(skb, DCB_ATTR_IFNAME, netdev->name))
                goto nla_put_failure;
        cee = nla_nest_start_noflag(skb, DCB_ATTR_CEE);
        if (!cee)
                goto nla_put_failure;

        /* local pg */
        if (ops->getpgtccfgtx && ops->getpgbwgcfgtx) {
                err = dcbnl_cee_pg_fill(skb, netdev, 1);
                if (err)
                        goto nla_put_failure;
        }

        if (ops->getpgtccfgrx && ops->getpgbwgcfgrx) {
                err = dcbnl_cee_pg_fill(skb, netdev, 0);
                if (err)
                        goto nla_put_failure;
        }

        /* local pfc */
        if (ops->getpfccfg) {
                struct nlattr *pfc_nest = nla_nest_start_noflag(skb,
                                                                DCB_ATTR_CEE_PFC);

                if (!pfc_nest)
                        goto nla_put_failure;

                for (i = DCB_PFC_UP_ATTR_0; i <= DCB_PFC_UP_ATTR_7; i++) {
                        ops->getpfccfg(netdev, i - DCB_PFC_UP_ATTR_0, &value);
                        if (nla_put_u8(skb, i, value))
                                goto nla_put_failure;
                }
                nla_nest_end(skb, pfc_nest);
        }

        /* local app */
        spin_lock_bh(&dcb_lock);
        app = nla_nest_start_noflag(skb, DCB_ATTR_CEE_APP_TABLE);
        if (!app)
                goto dcb_unlock;

        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->ifindex == netdev->ifindex) {
                        struct nlattr *app_nest = nla_nest_start_noflag(skb,
                                                                        DCB_ATTR_APP);
                        if (!app_nest)
                                goto dcb_unlock;

                        err = nla_put_u8(skb, DCB_APP_ATTR_IDTYPE,
                                         itr->app.selector);
                        if (err)
                                goto dcb_unlock;

                        err = nla_put_u16(skb, DCB_APP_ATTR_ID,
                                          itr->app.protocol);
                        if (err)
                                goto dcb_unlock;

                        err = nla_put_u8(skb, DCB_APP_ATTR_PRIORITY,
                                         itr->app.priority);
                        if (err)
                                goto dcb_unlock;

                        nla_nest_end(skb, app_nest);
                }
        }
        nla_nest_end(skb, app);

        if (netdev->dcbnl_ops->getdcbx)
                dcbx = netdev->dcbnl_ops->getdcbx(netdev);
        else
                dcbx = -EOPNOTSUPP;

        spin_unlock_bh(&dcb_lock);

        /* features flags */
        if (ops->getfeatcfg) {
                struct nlattr *feat = nla_nest_start_noflag(skb,
                                                            DCB_ATTR_CEE_FEAT);
                if (!feat)
                        goto nla_put_failure;

                for (i = DCB_FEATCFG_ATTR_ALL + 1; i <= DCB_FEATCFG_ATTR_MAX;
                     i++)
                        if (!ops->getfeatcfg(netdev, i, &value) &&
                            nla_put_u8(skb, i, value))
                                goto nla_put_failure;

                nla_nest_end(skb, feat);
        }

        /* peer info if available */
        if (ops->cee_peer_getpg) {
                struct cee_pg pg;
                memset(&pg, 0, sizeof(pg));
                err = ops->cee_peer_getpg(netdev, &pg);
                if (!err &&
                    nla_put(skb, DCB_ATTR_CEE_PEER_PG, sizeof(pg), &pg))
                        goto nla_put_failure;
        }

        if (ops->cee_peer_getpfc) {
                struct cee_pfc pfc;
                memset(&pfc, 0, sizeof(pfc));
                err = ops->cee_peer_getpfc(netdev, &pfc);
                if (!err &&
                    nla_put(skb, DCB_ATTR_CEE_PEER_PFC, sizeof(pfc), &pfc))
                        goto nla_put_failure;
        }

        if (ops->peer_getappinfo && ops->peer_getapptable) {
                err = dcbnl_build_peer_app(netdev, skb,
                                           DCB_ATTR_CEE_PEER_APP_TABLE,
                                           DCB_ATTR_CEE_PEER_APP_INFO,
                                           DCB_ATTR_CEE_PEER_APP);
                if (err)
                        goto nla_put_failure;
        }
        nla_nest_end(skb, cee);

        /* DCBX state */
        if (dcbx >= 0) {
                err = nla_put_u8(skb, DCB_ATTR_DCBX, dcbx);
                if (err)
                        goto nla_put_failure;
        }
        return 0;

dcb_unlock:
        spin_unlock_bh(&dcb_lock);
nla_put_failure:
        err = -EMSGSIZE;
        return err;
}

static int dcbnl_notify(struct net_device *dev, int event, int cmd,
                        u32 seq, u32 portid, int dcbx_ver)
{
        struct net *net = dev_net(dev);
        struct sk_buff *skb;
        struct nlmsghdr *nlh;
        const struct dcbnl_rtnl_ops *ops = dev->dcbnl_ops;
        int err;

        if (!ops)
                return -EOPNOTSUPP;

        skb = dcbnl_newmsg(event, cmd, portid, seq, 0, &nlh);
        if (!skb)
                return -ENOMEM;

        if (dcbx_ver == DCB_CAP_DCBX_VER_IEEE)
                err = dcbnl_ieee_fill(skb, dev);
        else
                err = dcbnl_cee_fill(skb, dev);

        if (err < 0) {
                /* Report error to broadcast listeners */
                nlmsg_free(skb);
                rtnl_set_sk_err(net, RTNLGRP_DCB, err);
        } else {
                /* End nlmsg and notify broadcast listeners */
                nlmsg_end(skb, nlh);
                rtnl_notify(skb, net, 0, RTNLGRP_DCB, NULL, GFP_KERNEL);
        }

        return err;
}

int dcbnl_ieee_notify(struct net_device *dev, int event, int cmd,
                      u32 seq, u32 portid)
{
        return dcbnl_notify(dev, event, cmd, seq, portid, DCB_CAP_DCBX_VER_IEEE);
}
EXPORT_SYMBOL(dcbnl_ieee_notify);

int dcbnl_cee_notify(struct net_device *dev, int event, int cmd,
                     u32 seq, u32 portid)
{
        return dcbnl_notify(dev, event, cmd, seq, portid, DCB_CAP_DCBX_VER_CEE);
}
EXPORT_SYMBOL(dcbnl_cee_notify);

/* Handle IEEE 802.1Qaz/802.1Qau/802.1Qbb SET commands.
 * If any requested operation can not be completed
 * the entire msg is aborted and error value is returned.
 * No attempt is made to reconcile the case where only part of the
 * cmd can be completed.
 */
static int dcbnl_ieee_set(struct net_device *netdev, struct nlmsghdr *nlh,
                          u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        struct nlattr *ieee[DCB_ATTR_IEEE_MAX + 1];
        int prio;
        int err;

        if (!ops)
                return -EOPNOTSUPP;

        if (!tb[DCB_ATTR_IEEE])
                return -EINVAL;

        err = nla_parse_nested_deprecated(ieee, DCB_ATTR_IEEE_MAX,
                                          tb[DCB_ATTR_IEEE],
                                          dcbnl_ieee_policy, NULL);
        if (err)
                return err;

        if (ieee[DCB_ATTR_IEEE_ETS] && ops->ieee_setets) {
                struct ieee_ets *ets = nla_data(ieee[DCB_ATTR_IEEE_ETS]);
                err = ops->ieee_setets(netdev, ets);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_IEEE_MAXRATE] && ops->ieee_setmaxrate) {
                struct ieee_maxrate *maxrate =
                        nla_data(ieee[DCB_ATTR_IEEE_MAXRATE]);
                err = ops->ieee_setmaxrate(netdev, maxrate);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_IEEE_QCN] && ops->ieee_setqcn) {
                struct ieee_qcn *qcn =
                        nla_data(ieee[DCB_ATTR_IEEE_QCN]);

                err = ops->ieee_setqcn(netdev, qcn);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_IEEE_PFC] && ops->ieee_setpfc) {
                struct ieee_pfc *pfc = nla_data(ieee[DCB_ATTR_IEEE_PFC]);
                err = ops->ieee_setpfc(netdev, pfc);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_DCB_BUFFER] && ops->dcbnl_setbuffer) {
                struct dcbnl_buffer *buffer =
                        nla_data(ieee[DCB_ATTR_DCB_BUFFER]);

                for (prio = 0; prio < ARRAY_SIZE(buffer->prio2buffer); prio++) {
                        if (buffer->prio2buffer[prio] >= DCBX_MAX_BUFFERS) {
                                err = -EINVAL;
                                goto err;
                        }
                }

                err = ops->dcbnl_setbuffer(netdev, buffer);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_DCB_REWR_TABLE]) {
                err = dcbnl_app_table_setdel(ieee[DCB_ATTR_DCB_REWR_TABLE],
                                             netdev,
                                             ops->dcbnl_setrewr ?: dcb_setrewr);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_IEEE_APP_TABLE]) {
                err = dcbnl_app_table_setdel(ieee[DCB_ATTR_IEEE_APP_TABLE],
                                             netdev, ops->ieee_setapp ?:
                                             dcb_ieee_setapp);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_DCB_APP_TRUST_TABLE]) {
                u8 selectors[IEEE_8021QAZ_APP_SEL_MAX + 1] = {0};
                struct nlattr *attr;
                int nselectors = 0;
                int rem;

                if (!ops->dcbnl_setapptrust) {
                        err = -EOPNOTSUPP;
                        goto err;
                }

                nla_for_each_nested(attr, ieee[DCB_ATTR_DCB_APP_TRUST_TABLE],
                                    rem) {
                        enum ieee_attrs_app type = nla_type(attr);
                        u8 selector;
                        int i;

                        if (!dcbnl_app_attr_type_validate(type) ||
                            nla_len(attr) != 1 ||
                            nselectors >= sizeof(selectors)) {
                                err = -EINVAL;
                                goto err;
                        }

                        selector = nla_get_u8(attr);

                        if (!dcbnl_app_selector_validate(type, selector)) {
                                err = -EINVAL;
                                goto err;
                        }

                        /* Duplicate selector ? */
                        for (i = 0; i < nselectors; i++) {
                                if (selectors[i] == selector) {
                                        err = -EINVAL;
                                        goto err;
                                }
                        }

                        selectors[nselectors++] = selector;
                }

                err = ops->dcbnl_setapptrust(netdev, selectors, nselectors);
                if (err)
                        goto err;
        }

err:
        err = nla_put_u8(skb, DCB_ATTR_IEEE, err);
        dcbnl_ieee_notify(netdev, RTM_SETDCB, DCB_CMD_IEEE_SET, seq, 0);
        return err;
}

static int dcbnl_ieee_get(struct net_device *netdev, struct nlmsghdr *nlh,
                          u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;

        if (!ops)
                return -EOPNOTSUPP;

        return dcbnl_ieee_fill(skb, netdev);
}

static int dcbnl_ieee_del(struct net_device *netdev, struct nlmsghdr *nlh,
                          u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;
        struct nlattr *ieee[DCB_ATTR_IEEE_MAX + 1];
        int err;

        if (!ops)
                return -EOPNOTSUPP;

        if (!tb[DCB_ATTR_IEEE])
                return -EINVAL;

        err = nla_parse_nested_deprecated(ieee, DCB_ATTR_IEEE_MAX,
                                          tb[DCB_ATTR_IEEE],
                                          dcbnl_ieee_policy, NULL);
        if (err)
                return err;

        if (ieee[DCB_ATTR_IEEE_APP_TABLE]) {
                err = dcbnl_app_table_setdel(ieee[DCB_ATTR_IEEE_APP_TABLE],
                                             netdev, ops->ieee_delapp ?:
                                             dcb_ieee_delapp);
                if (err)
                        goto err;
        }

        if (ieee[DCB_ATTR_DCB_REWR_TABLE]) {
                err = dcbnl_app_table_setdel(ieee[DCB_ATTR_DCB_REWR_TABLE],
                                             netdev,
                                             ops->dcbnl_delrewr ?: dcb_delrewr);
                if (err)
                        goto err;
        }

err:
        err = nla_put_u8(skb, DCB_ATTR_IEEE, err);
        dcbnl_ieee_notify(netdev, RTM_SETDCB, DCB_CMD_IEEE_DEL, seq, 0);
        return err;
}


/* DCBX configuration */
static int dcbnl_getdcbx(struct net_device *netdev, struct nlmsghdr *nlh,
                         u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        if (!netdev->dcbnl_ops->getdcbx)
                return -EOPNOTSUPP;

        return nla_put_u8(skb, DCB_ATTR_DCBX,
                          netdev->dcbnl_ops->getdcbx(netdev));
}

static int dcbnl_setdcbx(struct net_device *netdev, struct nlmsghdr *nlh,
                         u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        u8 value;

        if (!netdev->dcbnl_ops->setdcbx)
                return -EOPNOTSUPP;

        if (!tb[DCB_ATTR_DCBX])
                return -EINVAL;

        value = nla_get_u8(tb[DCB_ATTR_DCBX]);

        return nla_put_u8(skb, DCB_ATTR_DCBX,
                          netdev->dcbnl_ops->setdcbx(netdev, value));
}

static int dcbnl_getfeatcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                            u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_FEATCFG_ATTR_MAX + 1], *nest;
        u8 value;
        int ret, i;
        int getall = 0;

        if (!netdev->dcbnl_ops->getfeatcfg)
                return -EOPNOTSUPP;

        if (!tb[DCB_ATTR_FEATCFG])
                return -EINVAL;

        ret = nla_parse_nested_deprecated(data, DCB_FEATCFG_ATTR_MAX,
                                          tb[DCB_ATTR_FEATCFG],
                                          dcbnl_featcfg_nest, NULL);
        if (ret)
                return ret;

        nest = nla_nest_start_noflag(skb, DCB_ATTR_FEATCFG);
        if (!nest)
                return -EMSGSIZE;

        if (data[DCB_FEATCFG_ATTR_ALL])
                getall = 1;

        for (i = DCB_FEATCFG_ATTR_ALL+1; i <= DCB_FEATCFG_ATTR_MAX; i++) {
                if (!getall && !data[i])
                        continue;

                ret = netdev->dcbnl_ops->getfeatcfg(netdev, i, &value);
                if (!ret)
                        ret = nla_put_u8(skb, i, value);

                if (ret) {
                        nla_nest_cancel(skb, nest);
                        goto nla_put_failure;
                }
        }
        nla_nest_end(skb, nest);

nla_put_failure:
        return ret;
}

static int dcbnl_setfeatcfg(struct net_device *netdev, struct nlmsghdr *nlh,
                            u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        struct nlattr *data[DCB_FEATCFG_ATTR_MAX + 1];
        int ret, i;
        u8 value;

        if (!netdev->dcbnl_ops->setfeatcfg)
                return -ENOTSUPP;

        if (!tb[DCB_ATTR_FEATCFG])
                return -EINVAL;

        ret = nla_parse_nested_deprecated(data, DCB_FEATCFG_ATTR_MAX,
                                          tb[DCB_ATTR_FEATCFG],
                                          dcbnl_featcfg_nest, NULL);

        if (ret)
                goto err;

        for (i = DCB_FEATCFG_ATTR_ALL+1; i <= DCB_FEATCFG_ATTR_MAX; i++) {
                if (data[i] == NULL)
                        continue;

                value = nla_get_u8(data[i]);

                ret = netdev->dcbnl_ops->setfeatcfg(netdev, i, value);

                if (ret)
                        goto err;
        }
err:
        ret = nla_put_u8(skb, DCB_ATTR_FEATCFG, ret);

        return ret;
}

/* Handle CEE DCBX GET commands. */
static int dcbnl_cee_get(struct net_device *netdev, struct nlmsghdr *nlh,
                         u32 seq, struct nlattr **tb, struct sk_buff *skb)
{
        const struct dcbnl_rtnl_ops *ops = netdev->dcbnl_ops;

        if (!ops)
                return -EOPNOTSUPP;

        return dcbnl_cee_fill(skb, netdev);
}

struct reply_func {
        /* reply netlink message type */
        int        type;

        /* function to fill message contents */
        int   (*cb)(struct net_device *, struct nlmsghdr *, u32,
                    struct nlattr **, struct sk_buff *);
};

static const struct reply_func reply_funcs[DCB_CMD_MAX+1] = {
        [DCB_CMD_GSTATE]        = { RTM_GETDCB, dcbnl_getstate },
        [DCB_CMD_SSTATE]        = { RTM_SETDCB, dcbnl_setstate },
        [DCB_CMD_PFC_GCFG]        = { RTM_GETDCB, dcbnl_getpfccfg },
        [DCB_CMD_PFC_SCFG]        = { RTM_SETDCB, dcbnl_setpfccfg },
        [DCB_CMD_GPERM_HWADDR]        = { RTM_GETDCB, dcbnl_getperm_hwaddr },
        [DCB_CMD_GCAP]                = { RTM_GETDCB, dcbnl_getcap },
        [DCB_CMD_GNUMTCS]        = { RTM_GETDCB, dcbnl_getnumtcs },
        [DCB_CMD_SNUMTCS]        = { RTM_SETDCB, dcbnl_setnumtcs },
        [DCB_CMD_PFC_GSTATE]        = { RTM_GETDCB, dcbnl_getpfcstate },
        [DCB_CMD_PFC_SSTATE]        = { RTM_SETDCB, dcbnl_setpfcstate },
        [DCB_CMD_GAPP]                = { RTM_GETDCB, dcbnl_getapp },
        [DCB_CMD_SAPP]                = { RTM_SETDCB, dcbnl_setapp },
        [DCB_CMD_PGTX_GCFG]        = { RTM_GETDCB, dcbnl_pgtx_getcfg },
        [DCB_CMD_PGTX_SCFG]        = { RTM_SETDCB, dcbnl_pgtx_setcfg },
        [DCB_CMD_PGRX_GCFG]        = { RTM_GETDCB, dcbnl_pgrx_getcfg },
        [DCB_CMD_PGRX_SCFG]        = { RTM_SETDCB, dcbnl_pgrx_setcfg },
        [DCB_CMD_SET_ALL]        = { RTM_SETDCB, dcbnl_setall },
        [DCB_CMD_BCN_GCFG]        = { RTM_GETDCB, dcbnl_bcn_getcfg },
        [DCB_CMD_BCN_SCFG]        = { RTM_SETDCB, dcbnl_bcn_setcfg },
        [DCB_CMD_IEEE_GET]        = { RTM_GETDCB, dcbnl_ieee_get },
        [DCB_CMD_IEEE_SET]        = { RTM_SETDCB, dcbnl_ieee_set },
        [DCB_CMD_IEEE_DEL]        = { RTM_SETDCB, dcbnl_ieee_del },
        [DCB_CMD_GDCBX]                = { RTM_GETDCB, dcbnl_getdcbx },
        [DCB_CMD_SDCBX]                = { RTM_SETDCB, dcbnl_setdcbx },
        [DCB_CMD_GFEATCFG]        = { RTM_GETDCB, dcbnl_getfeatcfg },
        [DCB_CMD_SFEATCFG]        = { RTM_SETDCB, dcbnl_setfeatcfg },
        [DCB_CMD_CEE_GET]        = { RTM_GETDCB, dcbnl_cee_get },
};

static int dcb_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
                    struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct net_device *netdev;
        struct dcbmsg *dcb = nlmsg_data(nlh);
        struct nlattr *tb[DCB_ATTR_MAX + 1];
        u32 portid = NETLINK_CB(skb).portid;
        int ret = -EINVAL;
        struct sk_buff *reply_skb;
        struct nlmsghdr *reply_nlh = NULL;
        const struct reply_func *fn;

        if ((nlh->nlmsg_type == RTM_SETDCB) && !netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        ret = nlmsg_parse_deprecated(nlh, sizeof(*dcb), tb, DCB_ATTR_MAX,
                                     dcbnl_rtnl_policy, extack);
        if (ret < 0)
                return ret;

        if (dcb->cmd > DCB_CMD_MAX)
                return -EINVAL;

        /* check if a reply function has been defined for the command */
        fn = &reply_funcs[dcb->cmd];
        if (!fn->cb)
                return -EOPNOTSUPP;
        if (fn->type == RTM_SETDCB && !netlink_capable(skb, CAP_NET_ADMIN))
                return -EPERM;

        if (!tb[DCB_ATTR_IFNAME])
                return -EINVAL;

        netdev = __dev_get_by_name(net, nla_data(tb[DCB_ATTR_IFNAME]));
        if (!netdev)
                return -ENODEV;

        if (!netdev->dcbnl_ops)
                return -EOPNOTSUPP;

        reply_skb = dcbnl_newmsg(fn->type, dcb->cmd, portid, nlh->nlmsg_seq,
                                 nlh->nlmsg_flags, &reply_nlh);
        if (!reply_skb)
                return -ENOMEM;

        ret = fn->cb(netdev, nlh, nlh->nlmsg_seq, tb, reply_skb);
        if (ret < 0) {
                nlmsg_free(reply_skb);
                goto out;
        }

        nlmsg_end(reply_skb, reply_nlh);

        ret = rtnl_unicast(reply_skb, net, portid);
out:
        return ret;
}

static struct dcb_app_type *dcb_rewr_lookup(const struct dcb_app *app,
                                            int ifindex, int proto)
{
        struct dcb_app_type *itr;

        list_for_each_entry(itr, &dcb_rewr_list, list) {
                if (itr->app.selector == app->selector &&
                    itr->app.priority == app->priority &&
                    itr->ifindex == ifindex &&
                    ((proto == -1) || itr->app.protocol == proto))
                        return itr;
        }

        return NULL;
}

static struct dcb_app_type *dcb_app_lookup(const struct dcb_app *app,
                                           int ifindex, int prio)
{
        struct dcb_app_type *itr;

        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->app.selector == app->selector &&
                    itr->app.protocol == app->protocol &&
                    itr->ifindex == ifindex &&
                    ((prio == -1) || itr->app.priority == prio))
                        return itr;
        }

        return NULL;
}

static int dcb_app_add(struct list_head *list, const struct dcb_app *app,
                       int ifindex)
{
        struct dcb_app_type *entry;

        entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
        if (!entry)
                return -ENOMEM;

        memcpy(&entry->app, app, sizeof(*app));
        entry->ifindex = ifindex;
        list_add(&entry->list, list);

        return 0;
}

/**
 * dcb_getapp - retrieve the DCBX application user priority
 * @dev: network interface
 * @app: application to get user priority of
 *
 * On success returns a non-zero 802.1p user priority bitmap
 * otherwise returns 0 as the invalid user priority bitmap to
 * indicate an error.
 */
u8 dcb_getapp(struct net_device *dev, struct dcb_app *app)
{
        struct dcb_app_type *itr;
        u8 prio = 0;

        spin_lock_bh(&dcb_lock);
        itr = dcb_app_lookup(app, dev->ifindex, -1);
        if (itr)
                prio = itr->app.priority;
        spin_unlock_bh(&dcb_lock);

        return prio;
}
EXPORT_SYMBOL(dcb_getapp);

/**
 * dcb_setapp - add CEE dcb application data to app list
 * @dev: network interface
 * @new: application data to add
 *
 * Priority 0 is an invalid priority in CEE spec. This routine
 * removes applications from the app list if the priority is
 * set to zero. Priority is expected to be 8-bit 802.1p user priority bitmap
 */
int dcb_setapp(struct net_device *dev, struct dcb_app *new)
{
        struct dcb_app_type *itr;
        struct dcb_app_type event;
        int err = 0;

        event.ifindex = dev->ifindex;
        memcpy(&event.app, new, sizeof(event.app));
        if (dev->dcbnl_ops->getdcbx)
                event.dcbx = dev->dcbnl_ops->getdcbx(dev);

        spin_lock_bh(&dcb_lock);
        /* Search for existing match and replace */
        itr = dcb_app_lookup(new, dev->ifindex, -1);
        if (itr) {
                if (new->priority)
                        itr->app.priority = new->priority;
                else {
                        list_del(&itr->list);
                        kfree(itr);
                }
                goto out;
        }
        /* App type does not exist add new application type */
        if (new->priority)
                err = dcb_app_add(&dcb_app_list, new, dev->ifindex);
out:
        spin_unlock_bh(&dcb_lock);
        if (!err)
                call_dcbevent_notifiers(DCB_APP_EVENT, &event);
        return err;
}
EXPORT_SYMBOL(dcb_setapp);

/**
 * dcb_ieee_getapp_mask - retrieve the IEEE DCB application priority
 * @dev: network interface
 * @app: where to store the retrieve application data
 *
 * Helper routine which on success returns a non-zero 802.1Qaz user
 * priority bitmap otherwise returns 0 to indicate the dcb_app was
 * not found in APP list.
 */
u8 dcb_ieee_getapp_mask(struct net_device *dev, struct dcb_app *app)
{
        struct dcb_app_type *itr;
        u8 prio = 0;

        spin_lock_bh(&dcb_lock);
        itr = dcb_app_lookup(app, dev->ifindex, -1);
        if (itr)
                prio |= 1 << itr->app.priority;
        spin_unlock_bh(&dcb_lock);

        return prio;
}
EXPORT_SYMBOL(dcb_ieee_getapp_mask);

/* Get protocol value from rewrite entry. */
u16 dcb_getrewr(struct net_device *dev, struct dcb_app *app)
{
        struct dcb_app_type *itr;
        u16 proto = 0;

        spin_lock_bh(&dcb_lock);
        itr = dcb_rewr_lookup(app, dev->ifindex, -1);
        if (itr)
                proto = itr->app.protocol;
        spin_unlock_bh(&dcb_lock);

        return proto;
}
EXPORT_SYMBOL(dcb_getrewr);

 /* Add rewrite entry to the rewrite list. */
int dcb_setrewr(struct net_device *dev, struct dcb_app *new)
{
        int err;

        spin_lock_bh(&dcb_lock);
        /* Search for existing match and abort if found. */
        if (dcb_rewr_lookup(new, dev->ifindex, new->protocol)) {
                err = -EEXIST;
                goto out;
        }

        err = dcb_app_add(&dcb_rewr_list, new, dev->ifindex);
out:
        spin_unlock_bh(&dcb_lock);

        return err;
}
EXPORT_SYMBOL(dcb_setrewr);

/* Delete rewrite entry from the rewrite list. */
int dcb_delrewr(struct net_device *dev, struct dcb_app *del)
{
        struct dcb_app_type *itr;
        int err = -ENOENT;

        spin_lock_bh(&dcb_lock);
        /* Search for existing match and remove it. */
        itr = dcb_rewr_lookup(del, dev->ifindex, del->protocol);
        if (itr) {
                list_del(&itr->list);
                kfree(itr);
                err = 0;
        }

        spin_unlock_bh(&dcb_lock);

        return err;
}
EXPORT_SYMBOL(dcb_delrewr);

/**
 * dcb_ieee_setapp - add IEEE dcb application data to app list
 * @dev: network interface
 * @new: application data to add
 *
 * This adds Application data to the list. Multiple application
 * entries may exists for the same selector and protocol as long
 * as the priorities are different. Priority is expected to be a
 * 3-bit unsigned integer
 */
int dcb_ieee_setapp(struct net_device *dev, struct dcb_app *new)
{
        struct dcb_app_type event;
        int err = 0;

        event.ifindex = dev->ifindex;
        memcpy(&event.app, new, sizeof(event.app));
        if (dev->dcbnl_ops->getdcbx)
                event.dcbx = dev->dcbnl_ops->getdcbx(dev);

        spin_lock_bh(&dcb_lock);
        /* Search for existing match and abort if found */
        if (dcb_app_lookup(new, dev->ifindex, new->priority)) {
                err = -EEXIST;
                goto out;
        }

        err = dcb_app_add(&dcb_app_list, new, dev->ifindex);
out:
        spin_unlock_bh(&dcb_lock);
        if (!err)
                call_dcbevent_notifiers(DCB_APP_EVENT, &event);
        return err;
}
EXPORT_SYMBOL(dcb_ieee_setapp);

/**
 * dcb_ieee_delapp - delete IEEE dcb application data from list
 * @dev: network interface
 * @del: application data to delete
 *
 * This removes a matching APP data from the APP list
 */
int dcb_ieee_delapp(struct net_device *dev, struct dcb_app *del)
{
        struct dcb_app_type *itr;
        struct dcb_app_type event;
        int err = -ENOENT;

        event.ifindex = dev->ifindex;
        memcpy(&event.app, del, sizeof(event.app));
        if (dev->dcbnl_ops->getdcbx)
                event.dcbx = dev->dcbnl_ops->getdcbx(dev);

        spin_lock_bh(&dcb_lock);
        /* Search for existing match and remove it. */
        if ((itr = dcb_app_lookup(del, dev->ifindex, del->priority))) {
                list_del(&itr->list);
                kfree(itr);
                err = 0;
        }

        spin_unlock_bh(&dcb_lock);
        if (!err)
                call_dcbevent_notifiers(DCB_APP_EVENT, &event);
        return err;
}
EXPORT_SYMBOL(dcb_ieee_delapp);

/* dcb_getrewr_prio_pcp_mask_map - For a given device, find mapping from
 * priorities to the PCP and DEI values assigned to that priority.
 */
void dcb_getrewr_prio_pcp_mask_map(const struct net_device *dev,
                                   struct dcb_rewr_prio_pcp_map *p_map)
{
        int ifindex = dev->ifindex;
        struct dcb_app_type *itr;
        u8 prio;

        memset(p_map->map, 0, sizeof(p_map->map));

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_rewr_list, list) {
                if (itr->ifindex == ifindex &&
                    itr->app.selector == DCB_APP_SEL_PCP &&
                    itr->app.protocol < 16 &&
                    itr->app.priority < IEEE_8021QAZ_MAX_TCS) {
                        prio = itr->app.priority;
                        p_map->map[prio] |= 1 << itr->app.protocol;
                }
        }
        spin_unlock_bh(&dcb_lock);
}
EXPORT_SYMBOL(dcb_getrewr_prio_pcp_mask_map);

/* dcb_getrewr_prio_dscp_mask_map - For a given device, find mapping from
 * priorities to the DSCP values assigned to that priority.
 */
void dcb_getrewr_prio_dscp_mask_map(const struct net_device *dev,
                                    struct dcb_ieee_app_prio_map *p_map)
{
        int ifindex = dev->ifindex;
        struct dcb_app_type *itr;
        u8 prio;

        memset(p_map->map, 0, sizeof(p_map->map));

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_rewr_list, list) {
                if (itr->ifindex == ifindex &&
                    itr->app.selector == IEEE_8021QAZ_APP_SEL_DSCP &&
                    itr->app.protocol < 64 &&
                    itr->app.priority < IEEE_8021QAZ_MAX_TCS) {
                        prio = itr->app.priority;
                        p_map->map[prio] |= 1ULL << itr->app.protocol;
                }
        }
        spin_unlock_bh(&dcb_lock);
}
EXPORT_SYMBOL(dcb_getrewr_prio_dscp_mask_map);

/*
 * dcb_ieee_getapp_prio_dscp_mask_map - For a given device, find mapping from
 * priorities to the DSCP values assigned to that priority. Initialize p_map
 * such that each map element holds a bit mask of DSCP values configured for
 * that priority by APP entries.
 */
void dcb_ieee_getapp_prio_dscp_mask_map(const struct net_device *dev,
                                        struct dcb_ieee_app_prio_map *p_map)
{
        int ifindex = dev->ifindex;
        struct dcb_app_type *itr;
        u8 prio;

        memset(p_map->map, 0, sizeof(p_map->map));

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->ifindex == ifindex &&
                    itr->app.selector == IEEE_8021QAZ_APP_SEL_DSCP &&
                    itr->app.protocol < 64 &&
                    itr->app.priority < IEEE_8021QAZ_MAX_TCS) {
                        prio = itr->app.priority;
                        p_map->map[prio] |= 1ULL << itr->app.protocol;
                }
        }
        spin_unlock_bh(&dcb_lock);
}
EXPORT_SYMBOL(dcb_ieee_getapp_prio_dscp_mask_map);

/*
 * dcb_ieee_getapp_dscp_prio_mask_map - For a given device, find mapping from
 * DSCP values to the priorities assigned to that DSCP value. Initialize p_map
 * such that each map element holds a bit mask of priorities configured for a
 * given DSCP value by APP entries.
 */
void
dcb_ieee_getapp_dscp_prio_mask_map(const struct net_device *dev,
                                   struct dcb_ieee_app_dscp_map *p_map)
{
        int ifindex = dev->ifindex;
        struct dcb_app_type *itr;

        memset(p_map->map, 0, sizeof(p_map->map));

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->ifindex == ifindex &&
                    itr->app.selector == IEEE_8021QAZ_APP_SEL_DSCP &&
                    itr->app.protocol < 64 &&
                    itr->app.priority < IEEE_8021QAZ_MAX_TCS)
                        p_map->map[itr->app.protocol] |= 1 << itr->app.priority;
        }
        spin_unlock_bh(&dcb_lock);
}
EXPORT_SYMBOL(dcb_ieee_getapp_dscp_prio_mask_map);

/*
 * Per 802.1Q-2014, the selector value of 1 is used for matching on Ethernet
 * type, with valid PID values >= 1536. A special meaning is then assigned to
 * protocol value of 0: "default priority. For use when priority is not
 * otherwise specified".
 *
 * dcb_ieee_getapp_default_prio_mask - For a given device, find all APP entries
 * of the form {$PRIO, ETHERTYPE, 0} and construct a bit mask of all default
 * priorities set by these entries.
 */
u8 dcb_ieee_getapp_default_prio_mask(const struct net_device *dev)
{
        int ifindex = dev->ifindex;
        struct dcb_app_type *itr;
        u8 mask = 0;

        spin_lock_bh(&dcb_lock);
        list_for_each_entry(itr, &dcb_app_list, list) {
                if (itr->ifindex == ifindex &&
                    itr->app.selector == IEEE_8021QAZ_APP_SEL_ETHERTYPE &&
                    itr->app.protocol == 0 &&
                    itr->app.priority < IEEE_8021QAZ_MAX_TCS)
                        mask |= 1 << itr->app.priority;
        }
        spin_unlock_bh(&dcb_lock);

        return mask;
}
EXPORT_SYMBOL(dcb_ieee_getapp_default_prio_mask);

static void dcbnl_flush_dev(struct net_device *dev)
{
        struct dcb_app_type *itr, *tmp;

        spin_lock_bh(&dcb_lock);

        list_for_each_entry_safe(itr, tmp, &dcb_app_list, list) {
                if (itr->ifindex == dev->ifindex) {
                        list_del(&itr->list);
                        kfree(itr);
                }
        }

        spin_unlock_bh(&dcb_lock);
}

static int dcbnl_netdevice_event(struct notifier_block *nb,
                                 unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        switch (event) {
        case NETDEV_UNREGISTER:
                if (!dev->dcbnl_ops)
                        return NOTIFY_DONE;

                dcbnl_flush_dev(dev);

                return NOTIFY_OK;
        default:
                return NOTIFY_DONE;
        }
}

static struct notifier_block dcbnl_nb __read_mostly = {
        .notifier_call  = dcbnl_netdevice_event,
};

static const struct rtnl_msg_handler dcbnl_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_GETDCB, .doit = dcb_doit},
        {.msgtype = RTM_SETDCB, .doit = dcb_doit},
};

static int __init dcbnl_init(void)
{
        int err;

        err = register_netdevice_notifier(&dcbnl_nb);
        if (err)
                return err;

        rtnl_register_many(dcbnl_rtnl_msg_handlers);

        return 0;
}
device_initcall(dcbnl_init);




















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Based on arch/arm/include/asm/atomic.h
 *
 * Copyright (C) 1996 Russell King.
 * Copyright (C) 2002 Deep Blue Solutions Ltd.
 * Copyright (C) 2012 ARM Ltd.
 */

#ifndef __ASM_ATOMIC_LL_SC_H
#define __ASM_ATOMIC_LL_SC_H

#include <linux/stringify.h>

#ifndef CONFIG_CC_HAS_K_CONSTRAINT
#define K
#endif

/*
 * AArch64 UP and SMP safe atomic ops.  We use load exclusive and
 * store exclusive to ensure that these are atomic.  We may loop
 * to ensure that the update happens.
 */

#define ATOMIC_OP(op, asm_op, constraint)                                \
static __always_inline void                                                \
__ll_sc_atomic_##op(int i, atomic_t *v)                                        \
{                                                                        \
        unsigned long tmp;                                                \
        int result;                                                        \
                                                                        \
        asm volatile("// atomic_" #op "\n"                                \
        "        prfm        pstl1strm, %2\n"                                \
        "1:        ldxr        %w0, %2\n"                                        \
        "        " #asm_op "        %w0, %w0, %w3\n"                        \
        "        stxr        %w1, %w0, %2\n"                                        \
        "        cbnz        %w1, 1b\n"                                        \
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
        : __stringify(constraint) "r" (i));                                \
}

#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
static __always_inline int                                                \
__ll_sc_atomic_##op##_return##name(int i, atomic_t *v)                        \
{                                                                        \
        unsigned long tmp;                                                \
        int result;                                                        \
                                                                        \
        asm volatile("// atomic_" #op "_return" #name "\n"                \
        "        prfm        pstl1strm, %2\n"                                \
        "1:        ld" #acq "xr        %w0, %2\n"                                \
        "        " #asm_op "        %w0, %w0, %w3\n"                        \
        "        st" #rel "xr        %w1, %w0, %2\n"                                \
        "        cbnz        %w1, 1b\n"                                        \
        "        " #mb                                                        \
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
        : __stringify(constraint) "r" (i)                                \
        : cl);                                                                \
                                                                        \
        return result;                                                        \
}

#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
static __always_inline int                                                \
__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)                        \
{                                                                        \
        unsigned long tmp;                                                \
        int val, result;                                                \
                                                                        \
        asm volatile("// atomic_fetch_" #op #name "\n"                        \
        "        prfm        pstl1strm, %3\n"                                \
        "1:        ld" #acq "xr        %w0, %3\n"                                \
        "        " #asm_op "        %w1, %w0, %w4\n"                        \
        "        st" #rel "xr        %w2, %w1, %3\n"                                \
        "        cbnz        %w2, 1b\n"                                        \
        "        " #mb                                                        \
        : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)        \
        : __stringify(constraint) "r" (i)                                \
        : cl);                                                                \
                                                                        \
        return result;                                                        \
}

#define ATOMIC_OPS(...)                                                        \
        ATOMIC_OP(__VA_ARGS__)                                                \
        ATOMIC_OP_RETURN(        , dmb ish,  , l, "memory", __VA_ARGS__)\
        ATOMIC_OP_RETURN(_relaxed,        ,  ,  ,         , __VA_ARGS__)\
        ATOMIC_OP_RETURN(_acquire,        , a,  , "memory", __VA_ARGS__)\
        ATOMIC_OP_RETURN(_release,        ,  , l, "memory", __VA_ARGS__)\
        ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
        ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
        ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
        ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)

ATOMIC_OPS(add, add, I)
ATOMIC_OPS(sub, sub, J)

#undef ATOMIC_OPS
#define ATOMIC_OPS(...)                                                        \
        ATOMIC_OP(__VA_ARGS__)                                                \
        ATOMIC_FETCH_OP (        , dmb ish,  , l, "memory", __VA_ARGS__)\
        ATOMIC_FETCH_OP (_relaxed,        ,  ,  ,         , __VA_ARGS__)\
        ATOMIC_FETCH_OP (_acquire,        , a,  , "memory", __VA_ARGS__)\
        ATOMIC_FETCH_OP (_release,        ,  , l, "memory", __VA_ARGS__)

ATOMIC_OPS(and, and, K)
ATOMIC_OPS(or, orr, K)
ATOMIC_OPS(xor, eor, K)
/*
 * GAS converts the mysterious and undocumented BIC (immediate) alias to
 * an AND (immediate) instruction with the immediate inverted. We don't
 * have a constraint for this, so fall back to register.
 */
ATOMIC_OPS(andnot, bic, )

#undef ATOMIC_OPS
#undef ATOMIC_FETCH_OP
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP

#define ATOMIC64_OP(op, asm_op, constraint)                                \
static __always_inline void                                                \
__ll_sc_atomic64_##op(s64 i, atomic64_t *v)                                \
{                                                                        \
        s64 result;                                                        \
        unsigned long tmp;                                                \
                                                                        \
        asm volatile("// atomic64_" #op "\n"                                \
        "        prfm        pstl1strm, %2\n"                                \
        "1:        ldxr        %0, %2\n"                                        \
        "        " #asm_op "        %0, %0, %3\n"                                \
        "        stxr        %w1, %0, %2\n"                                        \
        "        cbnz        %w1, 1b"                                        \
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
        : __stringify(constraint) "r" (i));                                \
}

#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
static __always_inline long                                                \
__ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)                \
{                                                                        \
        s64 result;                                                        \
        unsigned long tmp;                                                \
                                                                        \
        asm volatile("// atomic64_" #op "_return" #name "\n"                \
        "        prfm        pstl1strm, %2\n"                                \
        "1:        ld" #acq "xr        %0, %2\n"                                \
        "        " #asm_op "        %0, %0, %3\n"                                \
        "        st" #rel "xr        %w1, %0, %2\n"                                \
        "        cbnz        %w1, 1b\n"                                        \
        "        " #mb                                                        \
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)                \
        : __stringify(constraint) "r" (i)                                \
        : cl);                                                                \
                                                                        \
        return result;                                                        \
}

#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
static __always_inline long                                                \
__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v)                        \
{                                                                        \
        s64 result, val;                                                \
        unsigned long tmp;                                                \
                                                                        \
        asm volatile("// atomic64_fetch_" #op #name "\n"                \
        "        prfm        pstl1strm, %3\n"                                \
        "1:        ld" #acq "xr        %0, %3\n"                                \
        "        " #asm_op "        %1, %0, %4\n"                                \
        "        st" #rel "xr        %w2, %1, %3\n"                                \
        "        cbnz        %w2, 1b\n"                                        \
        "        " #mb                                                        \
        : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)        \
        : __stringify(constraint) "r" (i)                                \
        : cl);                                                                \
                                                                        \
        return result;                                                        \
}

#define ATOMIC64_OPS(...)                                                \
        ATOMIC64_OP(__VA_ARGS__)                                        \
        ATOMIC64_OP_RETURN(, dmb ish,  , l, "memory", __VA_ARGS__)        \
        ATOMIC64_OP_RETURN(_relaxed,,  ,  ,         , __VA_ARGS__)        \
        ATOMIC64_OP_RETURN(_acquire,, a,  , "memory", __VA_ARGS__)        \
        ATOMIC64_OP_RETURN(_release,,  , l, "memory", __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)

ATOMIC64_OPS(add, add, I)
ATOMIC64_OPS(sub, sub, J)

#undef ATOMIC64_OPS
#define ATOMIC64_OPS(...)                                                \
        ATOMIC64_OP(__VA_ARGS__)                                        \
        ATOMIC64_FETCH_OP (, dmb ish,  , l, "memory", __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_relaxed,,  ,  ,         , __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_acquire,, a,  , "memory", __VA_ARGS__)        \
        ATOMIC64_FETCH_OP (_release,,  , l, "memory", __VA_ARGS__)

ATOMIC64_OPS(and, and, L)
ATOMIC64_OPS(or, orr, L)
ATOMIC64_OPS(xor, eor, L)
/*
 * GAS converts the mysterious and undocumented BIC (immediate) alias to
 * an AND (immediate) instruction with the immediate inverted. We don't
 * have a constraint for this, so fall back to register.
 */
ATOMIC64_OPS(andnot, bic, )

#undef ATOMIC64_OPS
#undef ATOMIC64_FETCH_OP
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP

static __always_inline s64
__ll_sc_atomic64_dec_if_positive(atomic64_t *v)
{
        s64 result;
        unsigned long tmp;

        asm volatile("// atomic64_dec_if_positive\n"
        "        prfm        pstl1strm, %2\n"
        "1:        ldxr        %0, %2\n"
        "        subs        %0, %0, #1\n"
        "        b.lt        2f\n"
        "        stlxr        %w1, %0, %2\n"
        "        cbnz        %w1, 1b\n"
        "        dmb        ish\n"
        "2:"
        : "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
        :
        : "cc", "memory");

        return result;
}

#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)        \
static __always_inline u##sz                                                \
__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,                        \
                                         unsigned long old,                \
                                         u##sz new)                        \
{                                                                        \
        unsigned long tmp;                                                \
        u##sz oldval;                                                        \
                                                                        \
        /*                                                                \
         * Sub-word sizes require explicit casting so that the compare  \
         * part of the cmpxchg doesn't end up interpreting non-zero        \
         * upper bits of the register containing "old".                        \
         */                                                                \
        if (sz < 32)                                                        \
                old = (u##sz)old;                                        \
                                                                        \
        asm volatile(                                                        \
        "        prfm        pstl1strm, %[v]\n"                                \
        "1:        ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"                \
        "        eor        %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"        \
        "        cbnz        %" #w "[tmp], 2f\n"                                \
        "        st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"        \
        "        cbnz        %w[tmp], 1b\n"                                        \
        "        " #mb "\n"                                                \
        "2:"                                                                \
        : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),                        \
          [v] "+Q" (*(u##sz *)ptr)                                        \
        : [old] __stringify(constraint) "r" (old), [new] "r" (new)        \
        : cl);                                                                \
                                                                        \
        return oldval;                                                        \
}

/*
 * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly
 * handle the 'K' constraint for the value 4294967295 - thus we use no
 * constraint for 32 bit operations.
 */
__CMPXCHG_CASE(w, b,     ,  8,        ,  ,  ,         , K)
__CMPXCHG_CASE(w, h,     , 16,        ,  ,  ,         , K)
__CMPXCHG_CASE(w,  ,     , 32,        ,  ,  ,         , K)
__CMPXCHG_CASE( ,  ,     , 64,        ,  ,  ,         , L)
__CMPXCHG_CASE(w, b, acq_,  8,        , a,  , "memory", K)
__CMPXCHG_CASE(w, h, acq_, 16,        , a,  , "memory", K)
__CMPXCHG_CASE(w,  , acq_, 32,        , a,  , "memory", K)
__CMPXCHG_CASE( ,  , acq_, 64,        , a,  , "memory", L)
__CMPXCHG_CASE(w, b, rel_,  8,        ,  , l, "memory", K)
__CMPXCHG_CASE(w, h, rel_, 16,        ,  , l, "memory", K)
__CMPXCHG_CASE(w,  , rel_, 32,        ,  , l, "memory", K)
__CMPXCHG_CASE( ,  , rel_, 64,        ,  , l, "memory", L)
__CMPXCHG_CASE(w, b,  mb_,  8, dmb ish,  , l, "memory", K)
__CMPXCHG_CASE(w, h,  mb_, 16, dmb ish,  , l, "memory", K)
__CMPXCHG_CASE(w,  ,  mb_, 32, dmb ish,  , l, "memory", K)
__CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)

#undef __CMPXCHG_CASE

union __u128_halves {
        u128 full;
        struct {
                u64 low, high;
        };
};

#define __CMPXCHG128(name, mb, rel, cl...)                             \
static __always_inline u128                                                \
__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new)        \
{                                                                        \
        union __u128_halves r, o = { .full = (old) },                        \
                               n = { .full = (new) };                        \
       unsigned int tmp;                                               \
                                                                        \
        asm volatile("// __cmpxchg128" #name "\n"                        \
       "       prfm    pstl1strm, %[v]\n"                              \
       "1:     ldxp    %[rl], %[rh], %[v]\n"                           \
       "       cmp     %[rl], %[ol]\n"                                 \
       "       ccmp    %[rh], %[oh], 0, eq\n"                          \
       "       b.ne    2f\n"                                           \
       "       st" #rel "xp    %w[tmp], %[nl], %[nh], %[v]\n"          \
       "       cbnz    %w[tmp], 1b\n"                                  \
        "        " #mb "\n"                                                \
        "2:"                                                                \
       : [v] "+Q" (*(u128 *)ptr),                                      \
         [rl] "=&r" (r.low), [rh] "=&r" (r.high),                      \
         [tmp] "=&r" (tmp)                                             \
       : [ol] "r" (o.low), [oh] "r" (o.high),                          \
         [nl] "r" (n.low), [nh] "r" (n.high)                           \
       : "cc", ##cl);                                                  \
                                                                        \
        return r.full;                                                        \
}

__CMPXCHG128(   ,        ,  )
__CMPXCHG128(_mb, dmb ish, l, "memory")

#undef __CMPXCHG128

#undef K

#endif        /* __ASM_ATOMIC_LL_SC_H */
































































































































































































































































  295 
  295 
























  293 



























































































































  294 
















































    4 



    4 


    4 








  293 



  294 


  293 












































  295 


























    4 





























































































































































































































































   29 






































































































































































































































































































































































































































































































































































































   29 




   29 
   29 



































   29 











   29 
   29 


   29 































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 */

#define pr_fmt(fmt)        "GICv3: " fmt

#include <linux/acpi.h>
#include <linux/cpu.h>
#include <linux/cpu_pm.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/irqdomain.h>
#include <linux/kernel.h>
#include <linux/kstrtox.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_irq.h>
#include <linux/percpu.h>
#include <linux/refcount.h>
#include <linux/slab.h>
#include <linux/iopoll.h>

#include <linux/irqchip.h>
#include <linux/irqchip/arm-gic-common.h>
#include <linux/irqchip/arm-gic-v3.h>
#include <linux/irqchip/arm-gic-v3-prio.h>
#include <linux/irqchip/irq-partition-percpu.h>
#include <linux/bitfield.h>
#include <linux/bits.h>
#include <linux/arm-smccc.h>

#include <asm/cputype.h>
#include <asm/exception.h>
#include <asm/smp_plat.h>
#include <asm/virt.h>

#include "irq-gic-common.h"

static u8 dist_prio_irq __ro_after_init = GICV3_PRIO_IRQ;
static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI;

#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996        (1ULL << 0)
#define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539        (1ULL << 1)
#define FLAGS_WORKAROUND_ASR_ERRATUM_8601001        (1ULL << 2)
#define FLAGS_WORKAROUND_INSECURE                (1ULL << 3)

#define GIC_IRQ_TYPE_PARTITION        (GIC_IRQ_TYPE_LPI + 1)

static struct cpumask broken_rdists __read_mostly __maybe_unused;

struct redist_region {
        void __iomem                *redist_base;
        phys_addr_t                phys_base;
        bool                        single_redist;
};

struct gic_chip_data {
        struct fwnode_handle        *fwnode;
        phys_addr_t                dist_phys_base;
        void __iomem                *dist_base;
        struct redist_region        *redist_regions;
        struct rdists                rdists;
        struct irq_domain        *domain;
        u64                        redist_stride;
        u32                        nr_redist_regions;
        u64                        flags;
        bool                        has_rss;
        unsigned int                ppi_nr;
        struct partition_desc        **ppi_descs;
};

#define T241_CHIPS_MAX                4
static void __iomem *t241_dist_base_alias[T241_CHIPS_MAX] __read_mostly;
static DEFINE_STATIC_KEY_FALSE(gic_nvidia_t241_erratum);

static DEFINE_STATIC_KEY_FALSE(gic_arm64_2941627_erratum);

static struct gic_chip_data gic_data __read_mostly;
static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key);

#define GIC_ID_NR        (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer))
#define GIC_LINE_NR        min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U)
#define GIC_ESPI_NR        GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer)

static bool nmi_support_forbidden;

/*
 * There are 16 SGIs, though we only actually use 8 in Linux. The other 8 SGIs
 * are potentially stolen by the secure side. Some code, especially code dealing
 * with hwirq IDs, is simplified by accounting for all 16.
 */
#define SGI_NR                16

/*
 * The behaviours of RPR and PMR registers differ depending on the value of
 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the
 * distributor and redistributors depends on whether security is enabled in the
 * GIC.
 *
 * When security is enabled, non-secure priority values from the (re)distributor
 * are presented to the GIC CPUIF as follow:
 *     (GIC_(R)DIST_PRI[irq] >> 1) | 0x80;
 *
 * If SCR_EL3.FIQ == 1, the values written to/read from PMR and RPR at non-secure
 * EL1 are subject to a similar operation thus matching the priorities presented
 * from the (re)distributor when security is enabled. When SCR_EL3.FIQ == 0,
 * these values are unchanged by the GIC.
 *
 * see GICv3/GICv4 Architecture Specification (IHI0069D):
 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt
 *   priorities.
 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1
 *   interrupt.
 */
static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis);

static u32 gic_get_pribits(void)
{
        u32 pribits;

        pribits = gic_read_ctlr();
        pribits &= ICC_CTLR_EL1_PRI_BITS_MASK;
        pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT;
        pribits++;

        return pribits;
}

static bool gic_has_group0(void)
{
        u32 val;
        u32 old_pmr;

        old_pmr = gic_read_pmr();

        /*
         * Let's find out if Group0 is under control of EL3 or not by
         * setting the highest possible, non-zero priority in PMR.
         *
         * If SCR_EL3.FIQ is set, the priority gets shifted down in
         * order for the CPU interface to set bit 7, and keep the
         * actual priority in the non-secure range. In the process, it
         * looses the least significant bit and the actual priority
         * becomes 0x80. Reading it back returns 0, indicating that
         * we're don't have access to Group0.
         */
        gic_write_pmr(BIT(8 - gic_get_pribits()));
        val = gic_read_pmr();

        gic_write_pmr(old_pmr);

        return val != 0;
}

static inline bool gic_dist_security_disabled(void)
{
        return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS;
}

static bool cpus_have_security_disabled __ro_after_init;
static bool cpus_have_group0 __ro_after_init;

static void __init gic_prio_init(void)
{
        bool ds;

        cpus_have_group0 = gic_has_group0();

        ds = gic_dist_security_disabled();
        if ((gic_data.flags & FLAGS_WORKAROUND_INSECURE) && !ds) {
                if (cpus_have_group0) {
                        u32 val;

                        val = readl_relaxed(gic_data.dist_base + GICD_CTLR);
                        val |= GICD_CTLR_DS;
                        writel_relaxed(val, gic_data.dist_base + GICD_CTLR);

                        ds = gic_dist_security_disabled();
                        if (ds)
                                pr_warn("Broken GIC integration, security disabled\n");
                } else {
                        pr_warn("Broken GIC integration, pNMI forbidden\n");
                        nmi_support_forbidden = true;
                }
        }

        cpus_have_security_disabled = ds;

        /*
         * How priority values are used by the GIC depends on two things:
         * the security state of the GIC (controlled by the GICD_CTRL.DS bit)
         * and if Group 0 interrupts can be delivered to Linux in the non-secure
         * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the
         * way priorities are presented in ICC_PMR_EL1 and in the distributor:
         *
         * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Distributor
         * -------------------------------------------------------
         *      1       |      -      |  unchanged  |  unchanged
         * -------------------------------------------------------
         *      0       |      1      |  non-secure |  non-secure
         * -------------------------------------------------------
         *      0       |      0      |  unchanged  |  non-secure
         *
         * In the non-secure view reads and writes are modified:
         *
         * - A value written is right-shifted by one and the MSB is set,
         *   forcing the priority into the non-secure range.
         *
         * - A value read is left-shifted by one.
         *
         * In the first two cases, where ICC_PMR_EL1 and the interrupt priority
         * are both either modified or unchanged, we can use the same set of
         * priorities.
         *
         * In the last case, where only the interrupt priorities are modified to
         * be in the non-secure range, we program the non-secure values into
         * the distributor to match the PMR values we want.
         */
        if (cpus_have_group0 && !cpus_have_security_disabled) {
                dist_prio_irq = __gicv3_prio_to_ns(dist_prio_irq);
                dist_prio_nmi = __gicv3_prio_to_ns(dist_prio_nmi);
        }

        pr_info("GICD_CTRL.DS=%d, SCR_EL3.FIQ=%d\n",
                cpus_have_security_disabled,
                !cpus_have_group0);
}

/* rdist_nmi_refs[n] == number of cpus having the rdist interrupt n set as NMI */
static refcount_t *rdist_nmi_refs;

static struct gic_kvm_info gic_v3_kvm_info __initdata;
static DEFINE_PER_CPU(bool, has_rss);

#define MPIDR_RS(mpidr)                        (((mpidr) & 0xF0UL) >> 4)
#define gic_data_rdist()                (this_cpu_ptr(gic_data.rdists.rdist))
#define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
#define gic_data_rdist_sgi_base()        (gic_data_rdist_rd_base() + SZ_64K)

/* Our default, arbitrary priority value. Linux only uses one anyway. */
#define DEFAULT_PMR_VALUE        0xf0

enum gic_intid_range {
        SGI_RANGE,
        PPI_RANGE,
        SPI_RANGE,
        EPPI_RANGE,
        ESPI_RANGE,
        LPI_RANGE,
        __INVALID_RANGE__
};

static enum gic_intid_range __get_intid_range(irq_hw_number_t hwirq)
{
        switch (hwirq) {
        case 0 ... 15:
                return SGI_RANGE;
        case 16 ... 31:
                return PPI_RANGE;
        case 32 ... 1019:
                return SPI_RANGE;
        case EPPI_BASE_INTID ... (EPPI_BASE_INTID + 63):
                return EPPI_RANGE;
        case ESPI_BASE_INTID ... (ESPI_BASE_INTID + 1023):
                return ESPI_RANGE;
        case 8192 ... GENMASK(23, 0):
                return LPI_RANGE;
        default:
                return __INVALID_RANGE__;
        }
}

static enum gic_intid_range get_intid_range(struct irq_data *d)
{
        return __get_intid_range(d->hwirq);
}

static inline bool gic_irq_in_rdist(struct irq_data *d)
{
        switch (get_intid_range(d)) {
        case SGI_RANGE:
        case PPI_RANGE:
        case EPPI_RANGE:
                return true;
        default:
                return false;
        }
}

static inline void __iomem *gic_dist_base_alias(struct irq_data *d)
{
        if (static_branch_unlikely(&gic_nvidia_t241_erratum)) {
                irq_hw_number_t hwirq = irqd_to_hwirq(d);
                u32 chip;

                /*
                 * For the erratum T241-FABRIC-4, read accesses to GICD_In{E}
                 * registers are directed to the chip that owns the SPI. The
                 * the alias region can also be used for writes to the
                 * GICD_In{E} except GICD_ICENABLERn. Each chip has support
                 * for 320 {E}SPIs. Mappings for all 4 chips:
                 *    Chip0 = 32-351
                 *    Chip1 = 352-671
                 *    Chip2 = 672-991
                 *    Chip3 = 4096-4415
                 */
                switch (__get_intid_range(hwirq)) {
                case SPI_RANGE:
                        chip = (hwirq - 32) / 320;
                        break;
                case ESPI_RANGE:
                        chip = 3;
                        break;
                default:
                        unreachable();
                }
                return t241_dist_base_alias[chip];
        }

        return gic_data.dist_base;
}

static inline void __iomem *gic_dist_base(struct irq_data *d)
{
        switch (get_intid_range(d)) {
        case SGI_RANGE:
        case PPI_RANGE:
        case EPPI_RANGE:
                /* SGI+PPI -> SGI_base for this CPU */
                return gic_data_rdist_sgi_base();

        case SPI_RANGE:
        case ESPI_RANGE:
                /* SPI -> dist_base */
                return gic_data.dist_base;

        default:
                return NULL;
        }
}

static void gic_do_wait_for_rwp(void __iomem *base, u32 bit)
{
        u32 val;
        int ret;

        ret = readl_relaxed_poll_timeout_atomic(base + GICD_CTLR, val, !(val & bit),
                                                1, USEC_PER_SEC);
        if (ret == -ETIMEDOUT)
                pr_err_ratelimited("RWP timeout, gone fishing\n");
}

/* Wait for completion of a distributor change */
static void gic_dist_wait_for_rwp(void)
{
        gic_do_wait_for_rwp(gic_data.dist_base, GICD_CTLR_RWP);
}

/* Wait for completion of a redistributor change */
static void gic_redist_wait_for_rwp(void)
{
        gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP);
}

static void gic_enable_redist(bool enable)
{
        void __iomem *rbase;
        u32 val;
        int ret;

        if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996)
                return;

        rbase = gic_data_rdist_rd_base();

        val = readl_relaxed(rbase + GICR_WAKER);
        if (enable)
                /* Wake up this CPU redistributor */
                val &= ~GICR_WAKER_ProcessorSleep;
        else
                val |= GICR_WAKER_ProcessorSleep;
        writel_relaxed(val, rbase + GICR_WAKER);

        if (!enable) {                /* Check that GICR_WAKER is writeable */
                val = readl_relaxed(rbase + GICR_WAKER);
                if (!(val & GICR_WAKER_ProcessorSleep))
                        return;        /* No PM support in this redistributor */
        }

        ret = readl_relaxed_poll_timeout_atomic(rbase + GICR_WAKER, val,
                                                enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep),
                                                1, USEC_PER_SEC);
        if (ret == -ETIMEDOUT) {
                pr_err_ratelimited("redistributor failed to %s...\n",
                                   enable ? "wakeup" : "sleep");
        }
}

/*
 * Routines to disable, enable, EOI and route interrupts
 */
static u32 convert_offset_index(struct irq_data *d, u32 offset, u32 *index)
{
        switch (get_intid_range(d)) {
        case SGI_RANGE:
        case PPI_RANGE:
        case SPI_RANGE:
                *index = d->hwirq;
                return offset;
        case EPPI_RANGE:
                /*
                 * Contrary to the ESPI range, the EPPI range is contiguous
                 * to the PPI range in the registers, so let's adjust the
                 * displacement accordingly. Consistency is overrated.
                 */
                *index = d->hwirq - EPPI_BASE_INTID + 32;
                return offset;
        case ESPI_RANGE:
                *index = d->hwirq - ESPI_BASE_INTID;
                switch (offset) {
                case GICD_ISENABLER:
                        return GICD_ISENABLERnE;
                case GICD_ICENABLER:
                        return GICD_ICENABLERnE;
                case GICD_ISPENDR:
                        return GICD_ISPENDRnE;
                case GICD_ICPENDR:
                        return GICD_ICPENDRnE;
                case GICD_ISACTIVER:
                        return GICD_ISACTIVERnE;
                case GICD_ICACTIVER:
                        return GICD_ICACTIVERnE;
                case GICD_IPRIORITYR:
                        return GICD_IPRIORITYRnE;
                case GICD_ICFGR:
                        return GICD_ICFGRnE;
                case GICD_IROUTER:
                        return GICD_IROUTERnE;
                default:
                        break;
                }
                break;
        default:
                break;
        }

        WARN_ON(1);
        *index = d->hwirq;
        return offset;
}

static int gic_peek_irq(struct irq_data *d, u32 offset)
{
        void __iomem *base;
        u32 index, mask;

        offset = convert_offset_index(d, offset, &index);
        mask = 1 << (index % 32);

        if (gic_irq_in_rdist(d))
                base = gic_data_rdist_sgi_base();
        else
                base = gic_dist_base_alias(d);

        return !!(readl_relaxed(base + offset + (index / 32) * 4) & mask);
}

static void gic_poke_irq(struct irq_data *d, u32 offset)
{
        void __iomem *base;
        u32 index, mask;

        offset = convert_offset_index(d, offset, &index);
        mask = 1 << (index % 32);

        if (gic_irq_in_rdist(d))
                base = gic_data_rdist_sgi_base();
        else
                base = gic_data.dist_base;

        writel_relaxed(mask, base + offset + (index / 32) * 4);
}

static void gic_mask_irq(struct irq_data *d)
{
        gic_poke_irq(d, GICD_ICENABLER);
        if (gic_irq_in_rdist(d))
                gic_redist_wait_for_rwp();
        else
                gic_dist_wait_for_rwp();
}

static void gic_eoimode1_mask_irq(struct irq_data *d)
{
        gic_mask_irq(d);
        /*
         * When masking a forwarded interrupt, make sure it is
         * deactivated as well.
         *
         * This ensures that an interrupt that is getting
         * disabled/masked will not get "stuck", because there is
         * noone to deactivate it (guest is being terminated).
         */
        if (irqd_is_forwarded_to_vcpu(d))
                gic_poke_irq(d, GICD_ICACTIVER);
}

static void gic_unmask_irq(struct irq_data *d)
{
        gic_poke_irq(d, GICD_ISENABLER);
}

static inline bool gic_supports_nmi(void)
{
        return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
               static_branch_likely(&supports_pseudo_nmis);
}

static int gic_irq_set_irqchip_state(struct irq_data *d,
                                     enum irqchip_irq_state which, bool val)
{
        u32 reg;

        if (d->hwirq >= 8192) /* SGI/PPI/SPI only */
                return -EINVAL;

        switch (which) {
        case IRQCHIP_STATE_PENDING:
                reg = val ? GICD_ISPENDR : GICD_ICPENDR;
                break;

        case IRQCHIP_STATE_ACTIVE:
                reg = val ? GICD_ISACTIVER : GICD_ICACTIVER;
                break;

        case IRQCHIP_STATE_MASKED:
                if (val) {
                        gic_mask_irq(d);
                        return 0;
                }
                reg = GICD_ISENABLER;
                break;

        default:
                return -EINVAL;
        }

        gic_poke_irq(d, reg);

        /*
         * Force read-back to guarantee that the active state has taken
         * effect, and won't race with a guest-driven deactivation.
         */
        if (reg == GICD_ISACTIVER)
                gic_peek_irq(d, reg);
        return 0;
}

static int gic_irq_get_irqchip_state(struct irq_data *d,
                                     enum irqchip_irq_state which, bool *val)
{
        if (d->hwirq >= 8192) /* PPI/SPI only */
                return -EINVAL;

        switch (which) {
        case IRQCHIP_STATE_PENDING:
                *val = gic_peek_irq(d, GICD_ISPENDR);
                break;

        case IRQCHIP_STATE_ACTIVE:
                *val = gic_peek_irq(d, GICD_ISACTIVER);
                break;

        case IRQCHIP_STATE_MASKED:
                *val = !gic_peek_irq(d, GICD_ISENABLER);
                break;

        default:
                return -EINVAL;
        }

        return 0;
}

static void gic_irq_set_prio(struct irq_data *d, u8 prio)
{
        void __iomem *base = gic_dist_base(d);
        u32 offset, index;

        offset = convert_offset_index(d, GICD_IPRIORITYR, &index);

        writeb_relaxed(prio, base + offset + index);
}

static u32 __gic_get_ppi_index(irq_hw_number_t hwirq)
{
        switch (__get_intid_range(hwirq)) {
        case PPI_RANGE:
                return hwirq - 16;
        case EPPI_RANGE:
                return hwirq - EPPI_BASE_INTID + 16;
        default:
                unreachable();
        }
}

static u32 __gic_get_rdist_index(irq_hw_number_t hwirq)
{
        switch (__get_intid_range(hwirq)) {
        case SGI_RANGE:
        case PPI_RANGE:
                return hwirq;
        case EPPI_RANGE:
                return hwirq - EPPI_BASE_INTID + 32;
        default:
                unreachable();
        }
}

static u32 gic_get_rdist_index(struct irq_data *d)
{
        return __gic_get_rdist_index(d->hwirq);
}

static int gic_irq_nmi_setup(struct irq_data *d)
{
        struct irq_desc *desc = irq_to_desc(d->irq);

        if (!gic_supports_nmi())
                return -EINVAL;

        if (gic_peek_irq(d, GICD_ISENABLER)) {
                pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
                return -EINVAL;
        }

        /*
         * A secondary irq_chip should be in charge of LPI request,
         * it should not be possible to get there
         */
        if (WARN_ON(irqd_to_hwirq(d) >= 8192))
                return -EINVAL;

        /* desc lock should already be held */
        if (gic_irq_in_rdist(d)) {
                u32 idx = gic_get_rdist_index(d);

                /*
                 * Setting up a percpu interrupt as NMI, only switch handler
                 * for first NMI
                 */
                if (!refcount_inc_not_zero(&rdist_nmi_refs[idx])) {
                        refcount_set(&rdist_nmi_refs[idx], 1);
                        desc->handle_irq = handle_percpu_devid_fasteoi_nmi;
                }
        } else {
                desc->handle_irq = handle_fasteoi_nmi;
        }

        gic_irq_set_prio(d, dist_prio_nmi);

        return 0;
}

static void gic_irq_nmi_teardown(struct irq_data *d)
{
        struct irq_desc *desc = irq_to_desc(d->irq);

        if (WARN_ON(!gic_supports_nmi()))
                return;

        if (gic_peek_irq(d, GICD_ISENABLER)) {
                pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq);
                return;
        }

        /*
         * A secondary irq_chip should be in charge of LPI request,
         * it should not be possible to get there
         */
        if (WARN_ON(irqd_to_hwirq(d) >= 8192))
                return;

        /* desc lock should already be held */
        if (gic_irq_in_rdist(d)) {
                u32 idx = gic_get_rdist_index(d);

                /* Tearing down NMI, only switch handler for last NMI */
                if (refcount_dec_and_test(&rdist_nmi_refs[idx]))
                        desc->handle_irq = handle_percpu_devid_irq;
        } else {
                desc->handle_irq = handle_fasteoi_irq;
        }

        gic_irq_set_prio(d, dist_prio_irq);
}

static bool gic_arm64_erratum_2941627_needed(struct irq_data *d)
{
        enum gic_intid_range range;

        if (!static_branch_unlikely(&gic_arm64_2941627_erratum))
                return false;

        range = get_intid_range(d);

        /*
         * The workaround is needed if the IRQ is an SPI and
         * the target cpu is different from the one we are
         * executing on.
         */
        return (range == SPI_RANGE || range == ESPI_RANGE) &&
                !cpumask_test_cpu(raw_smp_processor_id(),
                                  irq_data_get_effective_affinity_mask(d));
}

static void gic_eoi_irq(struct irq_data *d)
{
        write_gicreg(irqd_to_hwirq(d), ICC_EOIR1_EL1);
        isb();

        if (gic_arm64_erratum_2941627_needed(d)) {
                /*
                 * Make sure the GIC stream deactivate packet
                 * issued by ICC_EOIR1_EL1 has completed before
                 * deactivating through GICD_IACTIVER.
                 */
                dsb(sy);
                gic_poke_irq(d, GICD_ICACTIVER);
        }
}

static void gic_eoimode1_eoi_irq(struct irq_data *d)
{
        /*
         * No need to deactivate an LPI, or an interrupt that
         * is is getting forwarded to a vcpu.
         */
        if (irqd_to_hwirq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d))
                return;

        if (!gic_arm64_erratum_2941627_needed(d))
                gic_write_dir(irqd_to_hwirq(d));
        else
                gic_poke_irq(d, GICD_ICACTIVER);
}

static int gic_set_type(struct irq_data *d, unsigned int type)
{
        irq_hw_number_t irq = irqd_to_hwirq(d);
        enum gic_intid_range range;
        void __iomem *base;
        u32 offset, index;
        int ret;

        range = get_intid_range(d);

        /* Interrupt configuration for SGIs can't be changed */
        if (range == SGI_RANGE)
                return type != IRQ_TYPE_EDGE_RISING ? -EINVAL : 0;

        /* SPIs have restrictions on the supported types */
        if ((range == SPI_RANGE || range == ESPI_RANGE) &&
            type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
                return -EINVAL;

        if (gic_irq_in_rdist(d))
                base = gic_data_rdist_sgi_base();
        else
                base = gic_dist_base_alias(d);

        offset = convert_offset_index(d, GICD_ICFGR, &index);

        ret = gic_configure_irq(index, type, base + offset);
        if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) {
                /* Misconfigured PPIs are usually not fatal */
                pr_warn("GIC: PPI INTID%ld is secure or misconfigured\n", irq);
                ret = 0;
        }

        return ret;
}

static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu)
{
        if (get_intid_range(d) == SGI_RANGE)
                return -EINVAL;

        if (vcpu)
                irqd_set_forwarded_to_vcpu(d);
        else
                irqd_clr_forwarded_to_vcpu(d);
        return 0;
}

static u64 gic_cpu_to_affinity(int cpu)
{
        u64 mpidr = cpu_logical_map(cpu);
        u64 aff;

        /* ASR8601 needs to have its affinities shifted down... */
        if (unlikely(gic_data.flags & FLAGS_WORKAROUND_ASR_ERRATUM_8601001))
                mpidr = (MPIDR_AFFINITY_LEVEL(mpidr, 1)        |
                         (MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8));

        aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
               MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
               MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
               MPIDR_AFFINITY_LEVEL(mpidr, 0));

        return aff;
}

static void gic_deactivate_unhandled(u32 irqnr)
{
        if (static_branch_likely(&supports_deactivate_key)) {
                if (irqnr < 8192)
                        gic_write_dir(irqnr);
        } else {
                write_gicreg(irqnr, ICC_EOIR1_EL1);
                isb();
        }
}

/*
 * Follow a read of the IAR with any HW maintenance that needs to happen prior
 * to invoking the relevant IRQ handler. We must do two things:
 *
 * (1) Ensure instruction ordering between a read of IAR and subsequent
 *     instructions in the IRQ handler using an ISB.
 *
 *     It is possible for the IAR to report an IRQ which was signalled *after*
 *     the CPU took an IRQ exception as multiple interrupts can race to be
 *     recognized by the GIC, earlier interrupts could be withdrawn, and/or
 *     later interrupts could be prioritized by the GIC.
 *
 *     For devices which are tightly coupled to the CPU, such as PMUs, a
 *     context synchronization event is necessary to ensure that system
 *     register state is not stale, as these may have been indirectly written
 *     *after* exception entry.
 *
 * (2) Execute an interrupt priority drop when EOI mode 1 is in use.
 */
static inline void gic_complete_ack(u32 irqnr)
{
        if (static_branch_likely(&supports_deactivate_key))
                write_gicreg(irqnr, ICC_EOIR1_EL1);

        isb();
}

static bool gic_rpr_is_nmi_prio(void)
{
        if (!gic_supports_nmi())
                return false;

        return unlikely(gic_read_rpr() == GICV3_PRIO_NMI);
}

static bool gic_irqnr_is_special(u32 irqnr)
{
        return irqnr >= 1020 && irqnr <= 1023;
}

static void __gic_handle_irq(u32 irqnr, struct pt_regs *regs)
{
        if (gic_irqnr_is_special(irqnr))
                return;

        gic_complete_ack(irqnr);

        if (generic_handle_domain_irq(gic_data.domain, irqnr)) {
                WARN_ONCE(true, "Unexpected interrupt (irqnr %u)\n", irqnr);
                gic_deactivate_unhandled(irqnr);
        }
}

static void __gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
{
        if (gic_irqnr_is_special(irqnr))
                return;

        gic_complete_ack(irqnr);

        if (generic_handle_domain_nmi(gic_data.domain, irqnr)) {
                WARN_ONCE(true, "Unexpected pseudo-NMI (irqnr %u)\n", irqnr);
                gic_deactivate_unhandled(irqnr);
        }
}

/*
 * An exception has been taken from a context with IRQs enabled, and this could
 * be an IRQ or an NMI.
 *
 * The entry code called us with DAIF.IF set to keep NMIs masked. We must clear
 * DAIF.IF (and update ICC_PMR_EL1 to mask regular IRQs) prior to returning,
 * after handling any NMI but before handling any IRQ.
 *
 * The entry code has performed IRQ entry, and if an NMI is detected we must
 * perform NMI entry/exit around invoking the handler.
 */
static void __gic_handle_irq_from_irqson(struct pt_regs *regs)
{
        bool is_nmi;
        u32 irqnr;

        irqnr = gic_read_iar();

        is_nmi = gic_rpr_is_nmi_prio();

        if (is_nmi) {
                nmi_enter();
                __gic_handle_nmi(irqnr, regs);
                nmi_exit();
        }

        if (gic_prio_masking_enabled()) {
                gic_pmr_mask_irqs();
                gic_arch_enable_irqs();
        }

        if (!is_nmi)
                __gic_handle_irq(irqnr, regs);
}

/*
 * An exception has been taken from a context with IRQs disabled, which can only
 * be an NMI.
 *
 * The entry code called us with DAIF.IF set to keep NMIs masked. We must leave
 * DAIF.IF (and ICC_PMR_EL1) unchanged.
 *
 * The entry code has performed NMI entry.
 */
static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs)
{
        u64 pmr;
        u32 irqnr;

        /*
         * We were in a context with IRQs disabled. However, the
         * entry code has set PMR to a value that allows any
         * interrupt to be acknowledged, and not just NMIs. This can
         * lead to surprising effects if the NMI has been retired in
         * the meantime, and that there is an IRQ pending. The IRQ
         * would then be taken in NMI context, something that nobody
         * wants to debug twice.
         *
         * Until we sort this, drop PMR again to a level that will
         * actually only allow NMIs before reading IAR, and then
         * restore it to what it was.
         */
        pmr = gic_read_pmr();
        gic_pmr_mask_irqs();
        isb();
        irqnr = gic_read_iar();
        gic_write_pmr(pmr);

        __gic_handle_nmi(irqnr, regs);
}

static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
{
        if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs)))
                __gic_handle_irq_from_irqsoff(regs);
        else
                __gic_handle_irq_from_irqson(regs);
}

static void __init gic_dist_init(void)
{
        unsigned int i;
        u64 affinity;
        void __iomem *base = gic_data.dist_base;
        u32 val;

        /* Disable the distributor */
        writel_relaxed(0, base + GICD_CTLR);
        gic_dist_wait_for_rwp();

        /*
         * Configure SPIs as non-secure Group-1. This will only matter
         * if the GIC only has a single security state. This will not
         * do the right thing if the kernel is running in secure mode,
         * but that's not the intended use case anyway.
         */
        for (i = 32; i < GIC_LINE_NR; i += 32)
                writel_relaxed(~0, base + GICD_IGROUPR + i / 8);

        /* Extended SPI range, not handled by the GICv2/GICv3 common code */
        for (i = 0; i < GIC_ESPI_NR; i += 32) {
                writel_relaxed(~0U, base + GICD_ICENABLERnE + i / 8);
                writel_relaxed(~0U, base + GICD_ICACTIVERnE + i / 8);
        }

        for (i = 0; i < GIC_ESPI_NR; i += 32)
                writel_relaxed(~0U, base + GICD_IGROUPRnE + i / 8);

        for (i = 0; i < GIC_ESPI_NR; i += 16)
                writel_relaxed(0, base + GICD_ICFGRnE + i / 4);

        for (i = 0; i < GIC_ESPI_NR; i += 4)
                writel_relaxed(REPEAT_BYTE_U32(dist_prio_irq),
                               base + GICD_IPRIORITYRnE + i);

        /* Now do the common stuff */
        gic_dist_config(base, GIC_LINE_NR, dist_prio_irq);

        val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1;
        if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) {
                pr_info("Enabling SGIs without active state\n");
                val |= GICD_CTLR_nASSGIreq;
        }

        /* Enable distributor with ARE, Group1, and wait for it to drain */
        writel_relaxed(val, base + GICD_CTLR);
        gic_dist_wait_for_rwp();

        /*
         * Set all global interrupts to the boot CPU only. ARE must be
         * enabled.
         */
        affinity = gic_cpu_to_affinity(smp_processor_id());
        for (i = 32; i < GIC_LINE_NR; i++)
                gic_write_irouter(affinity, base + GICD_IROUTER + i * 8);

        for (i = 0; i < GIC_ESPI_NR; i++)
                gic_write_irouter(affinity, base + GICD_IROUTERnE + i * 8);
}

static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
{
        int ret = -ENODEV;
        int i;

        for (i = 0; i < gic_data.nr_redist_regions; i++) {
                void __iomem *ptr = gic_data.redist_regions[i].redist_base;
                u64 typer;
                u32 reg;

                reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
                if (reg != GIC_PIDR2_ARCH_GICv3 &&
                    reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */
                        pr_warn("No redistributor present @%p\n", ptr);
                        break;
                }

                do {
                        typer = gic_read_typer(ptr + GICR_TYPER);
                        ret = fn(gic_data.redist_regions + i, ptr);
                        if (!ret)
                                return 0;

                        if (gic_data.redist_regions[i].single_redist)
                                break;

                        if (gic_data.redist_stride) {
                                ptr += gic_data.redist_stride;
                        } else {
                                ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
                                if (typer & GICR_TYPER_VLPIS)
                                        ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
                        }
                } while (!(typer & GICR_TYPER_LAST));
        }

        return ret ? -ENODEV : 0;
}

static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
{
        unsigned long mpidr;
        u64 typer;
        u32 aff;

        /*
         * Convert affinity to a 32bit value that can be matched to
         * GICR_TYPER bits [63:32].
         */
        mpidr = gic_cpu_to_affinity(smp_processor_id());

        aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
               MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
               MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
               MPIDR_AFFINITY_LEVEL(mpidr, 0));

        typer = gic_read_typer(ptr + GICR_TYPER);
        if ((typer >> 32) == aff) {
                u64 offset = ptr - region->redist_base;
                raw_spin_lock_init(&gic_data_rdist()->rd_lock);
                gic_data_rdist_rd_base() = ptr;
                gic_data_rdist()->phys_base = region->phys_base + offset;

                pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
                        smp_processor_id(), mpidr,
                        (int)(region - gic_data.redist_regions),
                        &gic_data_rdist()->phys_base);
                return 0;
        }

        /* Try next one */
        return 1;
}

static int gic_populate_rdist(void)
{
        if (gic_iterate_rdists(__gic_populate_rdist) == 0)
                return 0;

        /* We couldn't even deal with ourselves... */
        WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
             smp_processor_id(),
             (unsigned long)cpu_logical_map(smp_processor_id()));
        return -ENODEV;
}

static int __gic_update_rdist_properties(struct redist_region *region,
                                         void __iomem *ptr)
{
        u64 typer = gic_read_typer(ptr + GICR_TYPER);
        u32 ctlr = readl_relaxed(ptr + GICR_CTLR);

        /* Boot-time cleanup */
        if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
                u64 val;

                /* Deactivate any present vPE */
                val = gicr_read_vpendbaser(ptr + SZ_128K + GICR_VPENDBASER);
                if (val & GICR_VPENDBASER_Valid)
                        gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
                                              ptr + SZ_128K + GICR_VPENDBASER);

                /* Mark the VPE table as invalid */
                val = gicr_read_vpropbaser(ptr + SZ_128K + GICR_VPROPBASER);
                val &= ~GICR_VPROPBASER_4_1_VALID;
                gicr_write_vpropbaser(val, ptr + SZ_128K + GICR_VPROPBASER);
        }

        gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS);

        /*
         * TYPER.RVPEID implies some form of DirectLPI, no matter what the
         * doc says... :-/ And CTLR.IR implies another subset of DirectLPI
         * that the ITS driver can make use of for LPIs (and not VLPIs).
         *
         * These are 3 different ways to express the same thing, depending
         * on the revision of the architecture and its relaxations over
         * time. Just group them under the 'direct_lpi' banner.
         */
        gic_data.rdists.has_rvpeid &= !!(typer & GICR_TYPER_RVPEID);
        gic_data.rdists.has_direct_lpi &= (!!(typer & GICR_TYPER_DirectLPIS) |
                                           !!(ctlr & GICR_CTLR_IR) |
                                           gic_data.rdists.has_rvpeid);
        gic_data.rdists.has_vpend_valid_dirty &= !!(typer & GICR_TYPER_DIRTY);

        /* Detect non-sensical configurations */
        if (WARN_ON_ONCE(gic_data.rdists.has_rvpeid && !gic_data.rdists.has_vlpis)) {
                gic_data.rdists.has_direct_lpi = false;
                gic_data.rdists.has_vlpis = false;
                gic_data.rdists.has_rvpeid = false;
        }

        gic_data.ppi_nr = min(GICR_TYPER_NR_PPIS(typer), gic_data.ppi_nr);

        return 1;
}

static void gic_update_rdist_properties(void)
{
        gic_data.ppi_nr = UINT_MAX;
        gic_iterate_rdists(__gic_update_rdist_properties);
        if (WARN_ON(gic_data.ppi_nr == UINT_MAX))
                gic_data.ppi_nr = 0;
        pr_info("GICv3 features: %d PPIs%s%s\n",
                gic_data.ppi_nr,
                gic_data.has_rss ? ", RSS" : "",
                gic_data.rdists.has_direct_lpi ? ", DirectLPI" : "");

        if (gic_data.rdists.has_vlpis)
                pr_info("GICv4 features: %s%s%s\n",
                        gic_data.rdists.has_direct_lpi ? "DirectLPI " : "",
                        gic_data.rdists.has_rvpeid ? "RVPEID " : "",
                        gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : "");
}

static void gic_cpu_sys_reg_enable(void)
{
        /*
         * Need to check that the SRE bit has actually been set. If
         * not, it means that SRE is disabled at EL2. We're going to
         * die painfully, and there is nothing we can do about it.
         *
         * Kindly inform the luser.
         */
        if (!gic_enable_sre())
                pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");

}

static void gic_cpu_sys_reg_init(void)
{
        int i, cpu = smp_processor_id();
        u64 mpidr = gic_cpu_to_affinity(cpu);
        u64 need_rss = MPIDR_RS(mpidr);
        bool group0;
        u32 pribits;

        pribits = gic_get_pribits();

        group0 = gic_has_group0();

        /* Set priority mask register */
        if (!gic_prio_masking_enabled()) {
                write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1);
        } else if (gic_supports_nmi()) {
                /*
                 * Check that all CPUs use the same priority space.
                 *
                 * If there's a mismatch with the boot CPU, the system is
                 * likely to die as interrupt masking will not work properly on
                 * all CPUs.
                 */
                WARN_ON(group0 != cpus_have_group0);
                WARN_ON(gic_dist_security_disabled() != cpus_have_security_disabled);
        }

        /*
         * Some firmwares hand over to the kernel with the BPR changed from
         * its reset value (and with a value large enough to prevent
         * any pre-emptive interrupts from working at all). Writing a zero
         * to BPR restores is reset value.
         */
        gic_write_bpr1(0);

        if (static_branch_likely(&supports_deactivate_key)) {
                /* EOI drops priority only (mode 1) */
                gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop);
        } else {
                /* EOI deactivates interrupt too (mode 0) */
                gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir);
        }

        /* Always whack Group0 before Group1 */
        if (group0) {
                switch(pribits) {
                case 8:
                case 7:
                        write_gicreg(0, ICC_AP0R3_EL1);
                        write_gicreg(0, ICC_AP0R2_EL1);
                        fallthrough;
                case 6:
                        write_gicreg(0, ICC_AP0R1_EL1);
                        fallthrough;
                case 5:
                case 4:
                        write_gicreg(0, ICC_AP0R0_EL1);
                }

                isb();
        }

        switch(pribits) {
        case 8:
        case 7:
                write_gicreg(0, ICC_AP1R3_EL1);
                write_gicreg(0, ICC_AP1R2_EL1);
                fallthrough;
        case 6:
                write_gicreg(0, ICC_AP1R1_EL1);
                fallthrough;
        case 5:
        case 4:
                write_gicreg(0, ICC_AP1R0_EL1);
        }

        isb();

        /* ... and let's hit the road... */
        gic_write_grpen1(1);

        /* Keep the RSS capability status in per_cpu variable */
        per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS);

        /* Check all the CPUs have capable of sending SGIs to other CPUs */
        for_each_online_cpu(i) {
                bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu);

                need_rss |= MPIDR_RS(gic_cpu_to_affinity(i));
                if (need_rss && (!have_rss))
                        pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n",
                                cpu, (unsigned long)mpidr,
                                i, (unsigned long)gic_cpu_to_affinity(i));
        }

        /**
         * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0,
         * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED
         * UNPREDICTABLE choice of :
         *   - The write is ignored.
         *   - The RS field is treated as 0.
         */
        if (need_rss && (!gic_data.has_rss))
                pr_crit_once("RSS is required but GICD doesn't support it\n");
}

static bool gicv3_nolpi;

static int __init gicv3_nolpi_cfg(char *buf)
{
        return kstrtobool(buf, &gicv3_nolpi);
}
early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg);

static int gic_dist_supports_lpis(void)
{
        return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) &&
                !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) &&
                !gicv3_nolpi);
}

static void gic_cpu_init(void)
{
        void __iomem *rbase;
        int i;

        /* Register ourselves with the rest of the world */
        if (gic_populate_rdist())
                return;

        gic_enable_redist(true);

        WARN((gic_data.ppi_nr > 16 || GIC_ESPI_NR != 0) &&
             !(gic_read_ctlr() & ICC_CTLR_EL1_ExtRange),
             "Distributor has extended ranges, but CPU%d doesn't\n",
             smp_processor_id());

        rbase = gic_data_rdist_sgi_base();

        /* Configure SGIs/PPIs as non-secure Group-1 */
        for (i = 0; i < gic_data.ppi_nr + SGI_NR; i += 32)
                writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8);

        gic_cpu_config(rbase, gic_data.ppi_nr + SGI_NR, dist_prio_irq);
        gic_redist_wait_for_rwp();

        /* initialise system registers */
        gic_cpu_sys_reg_init();
}

#ifdef CONFIG_SMP

#define MPIDR_TO_SGI_RS(mpidr)        (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT)
#define MPIDR_TO_SGI_CLUSTER_ID(mpidr)        ((mpidr) & ~0xFUL)

/*
 * gic_starting_cpu() is called after the last point where cpuhp is allowed
 * to fail. So pre check for problems earlier.
 */
static int gic_check_rdist(unsigned int cpu)
{
        if (cpumask_test_cpu(cpu, &broken_rdists))
                return -EINVAL;

        return 0;
}

static int gic_starting_cpu(unsigned int cpu)
{
        gic_cpu_sys_reg_enable();
        gic_cpu_init();

        if (gic_dist_supports_lpis())
                its_cpu_init();

        return 0;
}

static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
                                   unsigned long cluster_id)
{
        int next_cpu, cpu = *base_cpu;
        unsigned long mpidr;
        u16 tlist = 0;

        mpidr = gic_cpu_to_affinity(cpu);

        while (cpu < nr_cpu_ids) {
                tlist |= 1 << (mpidr & 0xf);

                next_cpu = cpumask_next(cpu, mask);
                if (next_cpu >= nr_cpu_ids)
                        goto out;
                cpu = next_cpu;

                mpidr = gic_cpu_to_affinity(cpu);

                if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) {
                        cpu--;
                        goto out;
                }
        }
out:
        *base_cpu = cpu;
        return tlist;
}

#define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \
        (MPIDR_AFFINITY_LEVEL(cluster_id, level) \
                << ICC_SGI1R_AFFINITY_## level ##_SHIFT)

static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
{
        u64 val;

        val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3)        |
               MPIDR_TO_SGI_AFFINITY(cluster_id, 2)        |
               irq << ICC_SGI1R_SGI_ID_SHIFT                |
               MPIDR_TO_SGI_AFFINITY(cluster_id, 1)        |
               MPIDR_TO_SGI_RS(cluster_id)                |
               tlist << ICC_SGI1R_TARGET_LIST_SHIFT);

        pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
        gic_write_sgi1r(val);
}

static void gic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
{
        int cpu;

        if (WARN_ON(d->hwirq >= 16))
                return;

        /*
         * Ensure that stores to Normal memory are visible to the
         * other CPUs before issuing the IPI.
         */
        dsb(ishst);

        for_each_cpu(cpu, mask) {
                u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(gic_cpu_to_affinity(cpu));
                u16 tlist;

                tlist = gic_compute_target_list(&cpu, mask, cluster_id);
                gic_send_sgi(cluster_id, tlist, d->hwirq);
        }

        /* Force the above writes to ICC_SGI1R_EL1 to be executed */
        isb();
}

static void __init gic_smp_init(void)
{
        struct irq_fwspec sgi_fwspec = {
                .fwnode                = gic_data.fwnode,
                .param_count        = 1,
        };
        int base_sgi;

        cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN,
                                  "irqchip/arm/gicv3:checkrdist",
                                  gic_check_rdist, NULL);

        cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING,
                                  "irqchip/arm/gicv3:starting",
                                  gic_starting_cpu, NULL);

        /* Register all 8 non-secure SGIs */
        base_sgi = irq_domain_alloc_irqs(gic_data.domain, 8, NUMA_NO_NODE, &sgi_fwspec);
        if (WARN_ON(base_sgi <= 0))
                return;

        set_smp_ipi_range(base_sgi, 8);
}

static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
                            bool force)
{
        unsigned int cpu;
        u32 offset, index;
        void __iomem *reg;
        int enabled;
        u64 val;

        if (force)
                cpu = cpumask_first(mask_val);
        else
                cpu = cpumask_any_and(mask_val, cpu_online_mask);

        if (cpu >= nr_cpu_ids)
                return -EINVAL;

        if (gic_irq_in_rdist(d))
                return -EINVAL;

        /* If interrupt was enabled, disable it first */
        enabled = gic_peek_irq(d, GICD_ISENABLER);
        if (enabled)
                gic_mask_irq(d);

        offset = convert_offset_index(d, GICD_IROUTER, &index);
        reg = gic_dist_base(d) + offset + (index * 8);
        val = gic_cpu_to_affinity(cpu);

        gic_write_irouter(val, reg);

        /*
         * If the interrupt was enabled, enabled it again. Otherwise,
         * just wait for the distributor to have digested our changes.
         */
        if (enabled)
                gic_unmask_irq(d);

        irq_data_update_effective_affinity(d, cpumask_of(cpu));

        return IRQ_SET_MASK_OK_DONE;
}
#else
#define gic_set_affinity        NULL
#define gic_ipi_send_mask        NULL
#define gic_smp_init()                do { } while(0)
#endif

static int gic_retrigger(struct irq_data *data)
{
        return !gic_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, true);
}

#ifdef CONFIG_CPU_PM
static int gic_cpu_pm_notifier(struct notifier_block *self,
                               unsigned long cmd, void *v)
{
        if (cmd == CPU_PM_EXIT || cmd == CPU_PM_ENTER_FAILED) {
                if (gic_dist_security_disabled())
                        gic_enable_redist(true);
                gic_cpu_sys_reg_enable();
                gic_cpu_sys_reg_init();
        } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) {
                gic_write_grpen1(0);
                gic_enable_redist(false);
        }
        return NOTIFY_OK;
}

static struct notifier_block gic_cpu_pm_notifier_block = {
        .notifier_call = gic_cpu_pm_notifier,
};

static void gic_cpu_pm_init(void)
{
        cpu_pm_register_notifier(&gic_cpu_pm_notifier_block);
}

#else
static inline void gic_cpu_pm_init(void) { }
#endif /* CONFIG_CPU_PM */

static struct irq_chip gic_chip = {
        .name                        = "GICv3",
        .irq_mask                = gic_mask_irq,
        .irq_unmask                = gic_unmask_irq,
        .irq_eoi                = gic_eoi_irq,
        .irq_set_type                = gic_set_type,
        .irq_set_affinity        = gic_set_affinity,
        .irq_retrigger          = gic_retrigger,
        .irq_get_irqchip_state        = gic_irq_get_irqchip_state,
        .irq_set_irqchip_state        = gic_irq_set_irqchip_state,
        .irq_nmi_setup                = gic_irq_nmi_setup,
        .irq_nmi_teardown        = gic_irq_nmi_teardown,
        .ipi_send_mask                = gic_ipi_send_mask,
        .flags                        = IRQCHIP_SET_TYPE_MASKED |
                                  IRQCHIP_SKIP_SET_WAKE |
                                  IRQCHIP_MASK_ON_SUSPEND,
};

static struct irq_chip gic_eoimode1_chip = {
        .name                        = "GICv3",
        .irq_mask                = gic_eoimode1_mask_irq,
        .irq_unmask                = gic_unmask_irq,
        .irq_eoi                = gic_eoimode1_eoi_irq,
        .irq_set_type                = gic_set_type,
        .irq_set_affinity        = gic_set_affinity,
        .irq_retrigger          = gic_retrigger,
        .irq_get_irqchip_state        = gic_irq_get_irqchip_state,
        .irq_set_irqchip_state        = gic_irq_set_irqchip_state,
        .irq_set_vcpu_affinity        = gic_irq_set_vcpu_affinity,
        .irq_nmi_setup                = gic_irq_nmi_setup,
        .irq_nmi_teardown        = gic_irq_nmi_teardown,
        .ipi_send_mask                = gic_ipi_send_mask,
        .flags                        = IRQCHIP_SET_TYPE_MASKED |
                                  IRQCHIP_SKIP_SET_WAKE |
                                  IRQCHIP_MASK_ON_SUSPEND,
};

static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
                              irq_hw_number_t hw)
{
        struct irq_chip *chip = &gic_chip;
        struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq));

        if (static_branch_likely(&supports_deactivate_key))
                chip = &gic_eoimode1_chip;

        switch (__get_intid_range(hw)) {
        case SGI_RANGE:
        case PPI_RANGE:
        case EPPI_RANGE:
                irq_set_percpu_devid(irq);
                irq_domain_set_info(d, irq, hw, chip, d->host_data,
                                    handle_percpu_devid_irq, NULL, NULL);
                break;

        case SPI_RANGE:
        case ESPI_RANGE:
                irq_domain_set_info(d, irq, hw, chip, d->host_data,
                                    handle_fasteoi_irq, NULL, NULL);
                irq_set_probe(irq);
                irqd_set_single_target(irqd);
                break;

        case LPI_RANGE:
                if (!gic_dist_supports_lpis())
                        return -EPERM;
                irq_domain_set_info(d, irq, hw, chip, d->host_data,
                                    handle_fasteoi_irq, NULL, NULL);
                break;

        default:
                return -EPERM;
        }

        /* Prevents SW retriggers which mess up the ACK/EOI ordering */
        irqd_set_handle_enforce_irqctx(irqd);
        return 0;
}

static int gic_irq_domain_translate(struct irq_domain *d,
                                    struct irq_fwspec *fwspec,
                                    unsigned long *hwirq,
                                    unsigned int *type)
{
        if (fwspec->param_count == 1 && fwspec->param[0] < 16) {
                *hwirq = fwspec->param[0];
                *type = IRQ_TYPE_EDGE_RISING;
                return 0;
        }

        if (is_of_node(fwspec->fwnode)) {
                if (fwspec->param_count < 3)
                        return -EINVAL;

                switch (fwspec->param[0]) {
                case 0:                        /* SPI */
                        *hwirq = fwspec->param[1] + 32;
                        break;
                case 1:                        /* PPI */
                        *hwirq = fwspec->param[1] + 16;
                        break;
                case 2:                        /* ESPI */
                        *hwirq = fwspec->param[1] + ESPI_BASE_INTID;
                        break;
                case 3:                        /* EPPI */
                        *hwirq = fwspec->param[1] + EPPI_BASE_INTID;
                        break;
                case GIC_IRQ_TYPE_LPI:        /* LPI */
                        *hwirq = fwspec->param[1];
                        break;
                case GIC_IRQ_TYPE_PARTITION:
                        *hwirq = fwspec->param[1];
                        if (fwspec->param[1] >= 16)
                                *hwirq += EPPI_BASE_INTID - 16;
                        else
                                *hwirq += 16;
                        break;
                default:
                        return -EINVAL;
                }

                *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;

                /*
                 * Make it clear that broken DTs are... broken.
                 * Partitioned PPIs are an unfortunate exception.
                 */
                WARN_ON(*type == IRQ_TYPE_NONE &&
                        fwspec->param[0] != GIC_IRQ_TYPE_PARTITION);
                return 0;
        }

        if (is_fwnode_irqchip(fwspec->fwnode)) {
                if(fwspec->param_count != 2)
                        return -EINVAL;

                if (fwspec->param[0] < 16) {
                        pr_err(FW_BUG "Illegal GSI%d translation request\n",
                               fwspec->param[0]);
                        return -EINVAL;
                }

                *hwirq = fwspec->param[0];
                *type = fwspec->param[1];

                WARN_ON(*type == IRQ_TYPE_NONE);
                return 0;
        }

        return -EINVAL;
}

static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
                                unsigned int nr_irqs, void *arg)
{
        int i, ret;
        irq_hw_number_t hwirq;
        unsigned int type = IRQ_TYPE_NONE;
        struct irq_fwspec *fwspec = arg;

        ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type);
        if (ret)
                return ret;

        for (i = 0; i < nr_irqs; i++) {
                ret = gic_irq_domain_map(domain, virq + i, hwirq + i);
                if (ret)
                        return ret;
        }

        return 0;
}

static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
                                unsigned int nr_irqs)
{
        int i;

        for (i = 0; i < nr_irqs; i++) {
                struct irq_data *d = irq_domain_get_irq_data(domain, virq + i);
                irq_set_handler(virq + i, NULL);
                irq_domain_reset_irq_data(d);
        }
}

static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec,
                                      irq_hw_number_t hwirq)
{
        enum gic_intid_range range;

        if (!gic_data.ppi_descs)
                return false;

        if (!is_of_node(fwspec->fwnode))
                return false;

        if (fwspec->param_count < 4 || !fwspec->param[3])
                return false;

        range = __get_intid_range(hwirq);
        if (range != PPI_RANGE && range != EPPI_RANGE)
                return false;

        return true;
}

static int gic_irq_domain_select(struct irq_domain *d,
                                 struct irq_fwspec *fwspec,
                                 enum irq_domain_bus_token bus_token)
{
        unsigned int type, ret, ppi_idx;
        irq_hw_number_t hwirq;

        /* Not for us */
        if (fwspec->fwnode != d->fwnode)
                return 0;

        /* Handle pure domain searches */
        if (!fwspec->param_count)
                return d->bus_token == bus_token;

        /* If this is not DT, then we have a single domain */
        if (!is_of_node(fwspec->fwnode))
                return 1;

        ret = gic_irq_domain_translate(d, fwspec, &hwirq, &type);
        if (WARN_ON_ONCE(ret))
                return 0;

        if (!fwspec_is_partitioned_ppi(fwspec, hwirq))
                return d == gic_data.domain;

        /*
         * If this is a PPI and we have a 4th (non-null) parameter,
         * then we need to match the partition domain.
         */
        ppi_idx = __gic_get_ppi_index(hwirq);
        return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]);
}

static const struct irq_domain_ops gic_irq_domain_ops = {
        .translate = gic_irq_domain_translate,
        .alloc = gic_irq_domain_alloc,
        .free = gic_irq_domain_free,
        .select = gic_irq_domain_select,
};

static int partition_domain_translate(struct irq_domain *d,
                                      struct irq_fwspec *fwspec,
                                      unsigned long *hwirq,
                                      unsigned int *type)
{
        unsigned long ppi_intid;
        struct device_node *np;
        unsigned int ppi_idx;
        int ret;

        if (!gic_data.ppi_descs)
                return -ENOMEM;

        np = of_find_node_by_phandle(fwspec->param[3]);
        if (WARN_ON(!np))
                return -EINVAL;

        ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type);
        if (WARN_ON_ONCE(ret))
                return 0;

        ppi_idx = __gic_get_ppi_index(ppi_intid);
        ret = partition_translate_id(gic_data.ppi_descs[ppi_idx],
                                     of_fwnode_handle(np));
        if (ret < 0)
                return ret;

        *hwirq = ret;
        *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;

        return 0;
}

static const struct irq_domain_ops partition_domain_ops = {
        .translate = partition_domain_translate,
        .select = gic_irq_domain_select,
};

static bool gic_enable_quirk_msm8996(void *data)
{
        struct gic_chip_data *d = data;

        d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996;

        return true;
}

static bool gic_enable_quirk_cavium_38539(void *data)
{
        struct gic_chip_data *d = data;

        d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;

        return true;
}

static bool gic_enable_quirk_hip06_07(void *data)
{
        struct gic_chip_data *d = data;

        /*
         * HIP06 GICD_IIDR clashes with GIC-600 product number (despite
         * not being an actual ARM implementation). The saving grace is
         * that GIC-600 doesn't have ESPI, so nothing to do in that case.
         * HIP07 doesn't even have a proper IIDR, and still pretends to
         * have ESPI. In both cases, put them right.
         */
        if (d->rdists.gicd_typer & GICD_TYPER_ESPI) {
                /* Zero both ESPI and the RES0 field next to it... */
                d->rdists.gicd_typer &= ~GENMASK(9, 8);
                return true;
        }

        return false;
}

#define T241_CHIPN_MASK                GENMASK_ULL(45, 44)
#define T241_CHIP_GICDA_OFFSET        0x1580000
#define SMCCC_SOC_ID_T241        0x036b0241

static bool gic_enable_quirk_nvidia_t241(void *data)
{
        s32 soc_id = arm_smccc_get_soc_id_version();
        unsigned long chip_bmask = 0;
        phys_addr_t phys;
        u32 i;

        /* Check JEP106 code for NVIDIA T241 chip (036b:0241) */
        if ((soc_id < 0) || (soc_id != SMCCC_SOC_ID_T241))
                return false;

        /* Find the chips based on GICR regions PHYS addr */
        for (i = 0; i < gic_data.nr_redist_regions; i++) {
                chip_bmask |= BIT(FIELD_GET(T241_CHIPN_MASK,
                                  (u64)gic_data.redist_regions[i].phys_base));
        }

        if (hweight32(chip_bmask) < 3)
                return false;

        /* Setup GICD alias regions */
        for (i = 0; i < ARRAY_SIZE(t241_dist_base_alias); i++) {
                if (chip_bmask & BIT(i)) {
                        phys = gic_data.dist_phys_base + T241_CHIP_GICDA_OFFSET;
                        phys |= FIELD_PREP(T241_CHIPN_MASK, i);
                        t241_dist_base_alias[i] = ioremap(phys, SZ_64K);
                        WARN_ON_ONCE(!t241_dist_base_alias[i]);
                }
        }
        static_branch_enable(&gic_nvidia_t241_erratum);
        return true;
}

static bool gic_enable_quirk_asr8601(void *data)
{
        struct gic_chip_data *d = data;

        d->flags |= FLAGS_WORKAROUND_ASR_ERRATUM_8601001;

        return true;
}

static bool gic_enable_quirk_arm64_2941627(void *data)
{
        static_branch_enable(&gic_arm64_2941627_erratum);
        return true;
}

static bool gic_enable_quirk_rk3399(void *data)
{
        struct gic_chip_data *d = data;

        if (of_machine_is_compatible("rockchip,rk3399")) {
                d->flags |= FLAGS_WORKAROUND_INSECURE;
                return true;
        }

        return false;
}

static bool rd_set_non_coherent(void *data)
{
        struct gic_chip_data *d = data;

        d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
        return true;
}

static const struct gic_quirk gic_quirks[] = {
        {
                .desc        = "GICv3: Qualcomm MSM8996 broken firmware",
                .compatible = "qcom,msm8996-gic-v3",
                .init        = gic_enable_quirk_msm8996,
        },
        {
                .desc        = "GICv3: ASR erratum 8601001",
                .compatible = "asr,asr8601-gic-v3",
                .init        = gic_enable_quirk_asr8601,
        },
        {
                .desc        = "GICv3: HIP06 erratum 161010803",
                .iidr        = 0x0204043b,
                .mask        = 0xffffffff,
                .init        = gic_enable_quirk_hip06_07,
        },
        {
                .desc        = "GICv3: HIP07 erratum 161010803",
                .iidr        = 0x00000000,
                .mask        = 0xffffffff,
                .init        = gic_enable_quirk_hip06_07,
        },
        {
                /*
                 * Reserved register accesses generate a Synchronous
                 * External Abort. This erratum applies to:
                 * - ThunderX: CN88xx
                 * - OCTEON TX: CN83xx, CN81xx
                 * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
                 */
                .desc        = "GICv3: Cavium erratum 38539",
                .iidr        = 0xa000034c,
                .mask        = 0xe8f00fff,
                .init        = gic_enable_quirk_cavium_38539,
        },
        {
                .desc        = "GICv3: NVIDIA erratum T241-FABRIC-4",
                .iidr        = 0x0402043b,
                .mask        = 0xffffffff,
                .init        = gic_enable_quirk_nvidia_t241,
        },
        {
                /*
                 * GIC-700: 2941627 workaround - IP variant [0,1]
                 *
                 */
                .desc        = "GICv3: ARM64 erratum 2941627",
                .iidr        = 0x0400043b,
                .mask        = 0xff0e0fff,
                .init        = gic_enable_quirk_arm64_2941627,
        },
        {
                /*
                 * GIC-700: 2941627 workaround - IP variant [2]
                 */
                .desc        = "GICv3: ARM64 erratum 2941627",
                .iidr        = 0x0402043b,
                .mask        = 0xff0f0fff,
                .init        = gic_enable_quirk_arm64_2941627,
        },
        {
                .desc   = "GICv3: non-coherent attribute",
                .property = "dma-noncoherent",
                .init   = rd_set_non_coherent,
        },
        {
                .desc        = "GICv3: Insecure RK3399 integration",
                .iidr        = 0x0000043b,
                .mask        = 0xff000fff,
                .init        = gic_enable_quirk_rk3399,
        },
        {
        }
};

static void gic_enable_nmi_support(void)
{
        int i;

        if (!gic_prio_masking_enabled() || nmi_support_forbidden)
                return;

        rdist_nmi_refs = kcalloc(gic_data.ppi_nr + SGI_NR,
                                 sizeof(*rdist_nmi_refs), GFP_KERNEL);
        if (!rdist_nmi_refs)
                return;

        for (i = 0; i < gic_data.ppi_nr + SGI_NR; i++)
                refcount_set(&rdist_nmi_refs[i], 0);

        pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n",
                gic_has_relaxed_pmr_sync() ? "relaxed" : "forced");

        static_branch_enable(&supports_pseudo_nmis);

        if (static_branch_likely(&supports_deactivate_key))
                gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI;
        else
                gic_chip.flags |= IRQCHIP_SUPPORTS_NMI;
}

static int __init gic_init_bases(phys_addr_t dist_phys_base,
                                 void __iomem *dist_base,
                                 struct redist_region *rdist_regs,
                                 u32 nr_redist_regions,
                                 u64 redist_stride,
                                 struct fwnode_handle *handle)
{
        u32 typer;
        int err;

        if (!is_hyp_mode_available())
                static_branch_disable(&supports_deactivate_key);

        if (static_branch_likely(&supports_deactivate_key))
                pr_info("GIC: Using split EOI/Deactivate mode\n");

        gic_data.fwnode = handle;
        gic_data.dist_phys_base = dist_phys_base;
        gic_data.dist_base = dist_base;
        gic_data.redist_regions = rdist_regs;
        gic_data.nr_redist_regions = nr_redist_regions;
        gic_data.redist_stride = redist_stride;

        /*
         * Find out how many interrupts are supported.
         */
        typer = readl_relaxed(gic_data.dist_base + GICD_TYPER);
        gic_data.rdists.gicd_typer = typer;

        gic_enable_quirks(readl_relaxed(gic_data.dist_base + GICD_IIDR),
                          gic_quirks, &gic_data);

        pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
        pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);

        /*
         * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
         * architecture spec (which says that reserved registers are RES0).
         */
        if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
                gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);

        gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
                                                 &gic_data);
        gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist));
        if (!static_branch_unlikely(&gic_nvidia_t241_erratum)) {
                /* Disable GICv4.x features for the erratum T241-FABRIC-4 */
                gic_data.rdists.has_rvpeid = true;
                gic_data.rdists.has_vlpis = true;
                gic_data.rdists.has_direct_lpi = true;
                gic_data.rdists.has_vpend_valid_dirty = true;
        }

        if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) {
                err = -ENOMEM;
                goto out_free;
        }

        irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED);

        gic_data.has_rss = !!(typer & GICD_TYPER_RSS);

        if (typer & GICD_TYPER_MBIS) {
                err = mbi_init(handle, gic_data.domain);
                if (err)
                        pr_err("Failed to initialize MBIs\n");
        }

        set_handle_irq(gic_handle_irq);

        gic_update_rdist_properties();

        gic_cpu_sys_reg_enable();
        gic_prio_init();
        gic_dist_init();
        gic_cpu_init();
        gic_enable_nmi_support();
        gic_smp_init();
        gic_cpu_pm_init();

        if (gic_dist_supports_lpis()) {
                its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq);
                its_cpu_init();
                its_lpi_memreserve_init();
        } else {
                if (IS_ENABLED(CONFIG_ARM_GIC_V2M))
                        gicv2m_init(handle, gic_data.domain);
        }

        return 0;

out_free:
        if (gic_data.domain)
                irq_domain_remove(gic_data.domain);
        free_percpu(gic_data.rdists.rdist);
        return err;
}

static int __init gic_validate_dist_version(void __iomem *dist_base)
{
        u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;

        if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4)
                return -ENODEV;

        return 0;
}

/* Create all possible partitions at boot time */
static void __init gic_populate_ppi_partitions(struct device_node *gic_node)
{
        struct device_node *parts_node, *child_part;
        int part_idx = 0, i;
        int nr_parts;
        struct partition_affinity *parts;

        parts_node = of_get_child_by_name(gic_node, "ppi-partitions");
        if (!parts_node)
                return;

        gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL);
        if (!gic_data.ppi_descs)
                goto out_put_node;

        nr_parts = of_get_child_count(parts_node);

        if (!nr_parts)
                goto out_put_node;

        parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL);
        if (WARN_ON(!parts))
                goto out_put_node;

        for_each_child_of_node(parts_node, child_part) {
                struct partition_affinity *part;
                int n;

                part = &parts[part_idx];

                part->partition_id = of_fwnode_handle(child_part);

                pr_info("GIC: PPI partition %pOFn[%d] { ",
                        child_part, part_idx);

                n = of_property_count_elems_of_size(child_part, "affinity",
                                                    sizeof(u32));
                WARN_ON(n <= 0);

                for (i = 0; i < n; i++) {
                        int err, cpu;
                        u32 cpu_phandle;
                        struct device_node *cpu_node;

                        err = of_property_read_u32_index(child_part, "affinity",
                                                         i, &cpu_phandle);
                        if (WARN_ON(err))
                                continue;

                        cpu_node = of_find_node_by_phandle(cpu_phandle);
                        if (WARN_ON(!cpu_node))
                                continue;

                        cpu = of_cpu_node_to_id(cpu_node);
                        if (WARN_ON(cpu < 0)) {
                                of_node_put(cpu_node);
                                continue;
                        }

                        pr_cont("%pOF[%d] ", cpu_node, cpu);

                        cpumask_set_cpu(cpu, &part->mask);
                        of_node_put(cpu_node);
                }

                pr_cont("}\n");
                part_idx++;
        }

        for (i = 0; i < gic_data.ppi_nr; i++) {
                unsigned int irq;
                struct partition_desc *desc;
                struct irq_fwspec ppi_fwspec = {
                        .fwnode                = gic_data.fwnode,
                        .param_count        = 3,
                        .param                = {
                                [0]        = GIC_IRQ_TYPE_PARTITION,
                                [1]        = i,
                                [2]        = IRQ_TYPE_NONE,
                        },
                };

                irq = irq_create_fwspec_mapping(&ppi_fwspec);
                if (WARN_ON(!irq))
                        continue;
                desc = partition_create_desc(gic_data.fwnode, parts, nr_parts,
                                             irq, &partition_domain_ops);
                if (WARN_ON(!desc))
                        continue;

                gic_data.ppi_descs[i] = desc;
        }

out_put_node:
        of_node_put(parts_node);
}

static void __init gic_of_setup_kvm_info(struct device_node *node, u32 nr_redist_regions)
{
        int ret;
        struct resource r;

        gic_v3_kvm_info.type = GIC_V3;

        gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
        if (!gic_v3_kvm_info.maint_irq)
                return;

        /* Also skip GICD, GICC, GICH */
        ret = of_address_to_resource(node, nr_redist_regions + 3, &r);
        if (!ret)
                gic_v3_kvm_info.vcpu = r;

        gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
        gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
        vgic_set_kvm_info(&gic_v3_kvm_info);
}

static void gic_request_region(resource_size_t base, resource_size_t size,
                               const char *name)
{
        if (!request_mem_region(base, size, name))
                pr_warn_once(FW_BUG "%s region %pa has overlapping address\n",
                             name, &base);
}

static void __iomem *gic_of_iomap(struct device_node *node, int idx,
                                  const char *name, struct resource *res)
{
        void __iomem *base;
        int ret;

        ret = of_address_to_resource(node, idx, res);
        if (ret)
                return IOMEM_ERR_PTR(ret);

        gic_request_region(res->start, resource_size(res), name);
        base = of_iomap(node, idx);

        return base ?: IOMEM_ERR_PTR(-ENOMEM);
}

static int __init gic_of_init(struct device_node *node, struct device_node *parent)
{
        phys_addr_t dist_phys_base;
        void __iomem *dist_base;
        struct redist_region *rdist_regs;
        struct resource res;
        u64 redist_stride;
        u32 nr_redist_regions;
        int err, i;

        dist_base = gic_of_iomap(node, 0, "GICD", &res);
        if (IS_ERR(dist_base)) {
                pr_err("%pOF: unable to map gic dist registers\n", node);
                return PTR_ERR(dist_base);
        }

        dist_phys_base = res.start;

        err = gic_validate_dist_version(dist_base);
        if (err) {
                pr_err("%pOF: no distributor detected, giving up\n", node);
                goto out_unmap_dist;
        }

        if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions))
                nr_redist_regions = 1;

        rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs),
                             GFP_KERNEL);
        if (!rdist_regs) {
                err = -ENOMEM;
                goto out_unmap_dist;
        }

        for (i = 0; i < nr_redist_regions; i++) {
                rdist_regs[i].redist_base = gic_of_iomap(node, 1 + i, "GICR", &res);
                if (IS_ERR(rdist_regs[i].redist_base)) {
                        pr_err("%pOF: couldn't map region %d\n", node, i);
                        err = -ENODEV;
                        goto out_unmap_rdist;
                }
                rdist_regs[i].phys_base = res.start;
        }

        if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
                redist_stride = 0;

        gic_enable_of_quirks(node, gic_quirks, &gic_data);

        err = gic_init_bases(dist_phys_base, dist_base, rdist_regs,
                             nr_redist_regions, redist_stride, &node->fwnode);
        if (err)
                goto out_unmap_rdist;

        gic_populate_ppi_partitions(node);

        if (static_branch_likely(&supports_deactivate_key))
                gic_of_setup_kvm_info(node, nr_redist_regions);
        return 0;

out_unmap_rdist:
        for (i = 0; i < nr_redist_regions; i++)
                if (rdist_regs[i].redist_base && !IS_ERR(rdist_regs[i].redist_base))
                        iounmap(rdist_regs[i].redist_base);
        kfree(rdist_regs);
out_unmap_dist:
        iounmap(dist_base);
        return err;
}

IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);

#ifdef CONFIG_ACPI
static struct
{
        void __iomem *dist_base;
        struct redist_region *redist_regs;
        u32 nr_redist_regions;
        bool single_redist;
        int enabled_rdists;
        u32 maint_irq;
        int maint_irq_mode;
        phys_addr_t vcpu_base;
} acpi_data __initdata;

static void __init
gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base)
{
        static int count = 0;

        acpi_data.redist_regs[count].phys_base = phys_base;
        acpi_data.redist_regs[count].redist_base = redist_base;
        acpi_data.redist_regs[count].single_redist = acpi_data.single_redist;
        count++;
}

static int __init
gic_acpi_parse_madt_redist(union acpi_subtable_headers *header,
                           const unsigned long end)
{
        struct acpi_madt_generic_redistributor *redist =
                        (struct acpi_madt_generic_redistributor *)header;
        void __iomem *redist_base;

        redist_base = ioremap(redist->base_address, redist->length);
        if (!redist_base) {
                pr_err("Couldn't map GICR region @%llx\n", redist->base_address);
                return -ENOMEM;
        }

        if (acpi_get_madt_revision() >= 7 &&
            (redist->flags & ACPI_MADT_GICR_NON_COHERENT))
                gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;

        gic_request_region(redist->base_address, redist->length, "GICR");

        gic_acpi_register_redist(redist->base_address, redist_base);
        return 0;
}

static int __init
gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header,
                         const unsigned long end)
{
        struct acpi_madt_generic_interrupt *gicc =
                                (struct acpi_madt_generic_interrupt *)header;
        u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
        u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2;
        void __iomem *redist_base;

        /* Neither enabled or online capable means it doesn't exist, skip it */
        if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
                return 0;

        /*
         * Capable but disabled CPUs can be brought online later. What about
         * the redistributor? ACPI doesn't want to say!
         * Virtual hotplug systems can use the MADT's "always-on" GICR entries.
         * Otherwise, prevent such CPUs from being brought online.
         */
        if (!(gicc->flags & ACPI_MADT_ENABLED)) {
                int cpu = get_cpu_for_acpi_id(gicc->uid);

                pr_warn("CPU %u's redistributor is inaccessible: this CPU can't be brought online\n", cpu);
                if (cpu >= 0)
                        cpumask_set_cpu(cpu, &broken_rdists);
                return 0;
        }

        redist_base = ioremap(gicc->gicr_base_address, size);
        if (!redist_base)
                return -ENOMEM;
        gic_request_region(gicc->gicr_base_address, size, "GICR");

        if (acpi_get_madt_revision() >= 7 &&
            (gicc->flags & ACPI_MADT_GICC_NON_COHERENT))
                gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;

        gic_acpi_register_redist(gicc->gicr_base_address, redist_base);
        return 0;
}

static int __init gic_acpi_collect_gicr_base(void)
{
        acpi_tbl_entry_handler redist_parser;
        enum acpi_madt_type type;

        if (acpi_data.single_redist) {
                type = ACPI_MADT_TYPE_GENERIC_INTERRUPT;
                redist_parser = gic_acpi_parse_madt_gicc;
        } else {
                type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR;
                redist_parser = gic_acpi_parse_madt_redist;
        }

        /* Collect redistributor base addresses in GICR entries */
        if (acpi_table_parse_madt(type, redist_parser, 0) > 0)
                return 0;

        pr_info("No valid GICR entries exist\n");
        return -ENODEV;
}

static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header,
                                  const unsigned long end)
{
        /* Subtable presence means that redist exists, that's it */
        return 0;
}

static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header,
                                      const unsigned long end)
{
        struct acpi_madt_generic_interrupt *gicc =
                                (struct acpi_madt_generic_interrupt *)header;

        /*
         * If GICC is enabled and has valid gicr base address, then it means
         * GICR base is presented via GICC. The redistributor is only known to
         * be accessible if the GICC is marked as enabled. If this bit is not
         * set, we'd need to add the redistributor at runtime, which isn't
         * supported.
         */
        if (gicc->flags & ACPI_MADT_ENABLED && gicc->gicr_base_address)
                acpi_data.enabled_rdists++;

        return 0;
}

static int __init gic_acpi_count_gicr_regions(void)
{
        int count;

        /*
         * Count how many redistributor regions we have. It is not allowed
         * to mix redistributor description, GICR and GICC subtables have to be
         * mutually exclusive.
         */
        count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR,
                                      gic_acpi_match_gicr, 0);
        if (count > 0) {
                acpi_data.single_redist = false;
                return count;
        }

        count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                                      gic_acpi_match_gicc, 0);
        if (count > 0) {
                acpi_data.single_redist = true;
                count = acpi_data.enabled_rdists;
        }

        return count;
}

static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header,
                                           struct acpi_probe_entry *ape)
{
        struct acpi_madt_generic_distributor *dist;
        int count;

        dist = (struct acpi_madt_generic_distributor *)header;
        if (dist->version != ape->driver_data)
                return false;

        /* We need to do that exercise anyway, the sooner the better */
        count = gic_acpi_count_gicr_regions();
        if (count <= 0)
                return false;

        acpi_data.nr_redist_regions = count;
        return true;
}

static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
                                                const unsigned long end)
{
        struct acpi_madt_generic_interrupt *gicc =
                (struct acpi_madt_generic_interrupt *)header;
        int maint_irq_mode;
        static int first_madt = true;

        if (!(gicc->flags &
              (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
                return 0;

        maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ?
                ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE;

        if (first_madt) {
                first_madt = false;

                acpi_data.maint_irq = gicc->vgic_interrupt;
                acpi_data.maint_irq_mode = maint_irq_mode;
                acpi_data.vcpu_base = gicc->gicv_base_address;

                return 0;
        }

        /*
         * The maintenance interrupt and GICV should be the same for every CPU
         */
        if ((acpi_data.maint_irq != gicc->vgic_interrupt) ||
            (acpi_data.maint_irq_mode != maint_irq_mode) ||
            (acpi_data.vcpu_base != gicc->gicv_base_address))
                return -EINVAL;

        return 0;
}

static bool __init gic_acpi_collect_virt_info(void)
{
        int count;

        count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
                                      gic_acpi_parse_virt_madt_gicc, 0);

        return (count > 0);
}

#define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K)
#define ACPI_GICV2_VCTRL_MEM_SIZE        (SZ_4K)
#define ACPI_GICV2_VCPU_MEM_SIZE        (SZ_8K)

static void __init gic_acpi_setup_kvm_info(void)
{
        int irq;

        if (!gic_acpi_collect_virt_info()) {
                pr_warn("Unable to get hardware information used for virtualization\n");
                return;
        }

        gic_v3_kvm_info.type = GIC_V3;

        irq = acpi_register_gsi(NULL, acpi_data.maint_irq,
                                acpi_data.maint_irq_mode,
                                ACPI_ACTIVE_HIGH);
        if (irq <= 0)
                return;

        gic_v3_kvm_info.maint_irq = irq;

        if (acpi_data.vcpu_base) {
                struct resource *vcpu = &gic_v3_kvm_info.vcpu;

                vcpu->flags = IORESOURCE_MEM;
                vcpu->start = acpi_data.vcpu_base;
                vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1;
        }

        gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis;
        gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid;
        vgic_set_kvm_info(&gic_v3_kvm_info);
}

static struct fwnode_handle *gsi_domain_handle;

static struct fwnode_handle *gic_v3_get_gsi_domain_id(u32 gsi)
{
        return gsi_domain_handle;
}

static int __init
gic_acpi_init(union acpi_subtable_headers *header, const unsigned long end)
{
        struct acpi_madt_generic_distributor *dist;
        size_t size;
        int i, err;

        /* Get distributor base address */
        dist = (struct acpi_madt_generic_distributor *)header;
        acpi_data.dist_base = ioremap(dist->base_address,
                                      ACPI_GICV3_DIST_MEM_SIZE);
        if (!acpi_data.dist_base) {
                pr_err("Unable to map GICD registers\n");
                return -ENOMEM;
        }
        gic_request_region(dist->base_address, ACPI_GICV3_DIST_MEM_SIZE, "GICD");

        err = gic_validate_dist_version(acpi_data.dist_base);
        if (err) {
                pr_err("No distributor detected at @%p, giving up\n",
                       acpi_data.dist_base);
                goto out_dist_unmap;
        }

        size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions;
        acpi_data.redist_regs = kzalloc(size, GFP_KERNEL);
        if (!acpi_data.redist_regs) {
                err = -ENOMEM;
                goto out_dist_unmap;
        }

        err = gic_acpi_collect_gicr_base();
        if (err)
                goto out_redist_unmap;

        gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address);
        if (!gsi_domain_handle) {
                err = -ENOMEM;
                goto out_redist_unmap;
        }

        err = gic_init_bases(dist->base_address, acpi_data.dist_base,
                             acpi_data.redist_regs, acpi_data.nr_redist_regions,
                             0, gsi_domain_handle);
        if (err)
                goto out_fwhandle_free;

        acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v3_get_gsi_domain_id);

        if (static_branch_likely(&supports_deactivate_key))
                gic_acpi_setup_kvm_info();

        return 0;

out_fwhandle_free:
        irq_domain_free_fwnode(gsi_domain_handle);
out_redist_unmap:
        for (i = 0; i < acpi_data.nr_redist_regions; i++)
                if (acpi_data.redist_regs[i].redist_base)
                        iounmap(acpi_data.redist_regs[i].redist_base);
        kfree(acpi_data.redist_regs);
out_dist_unmap:
        iounmap(acpi_data.dist_base);
        return err;
}
IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
                     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3,
                     gic_acpi_init);
IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
                     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4,
                     gic_acpi_init);
IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR,
                     acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE,
                     gic_acpi_init);
#endif




































































































  150 














  150 














    1 









  149 












   65 






   92 
    1 














  152 





    1 






  152 































































































































































































































































































































































































   89 










   89 























   89 




   89 


   89 











   89 






































































































































































































































































































  152 







  153 






  152 















  153 















  153 





































































































































































































  153 














   89 










  152 



  153 


  153 













   89 
   89 








































































































































































































































































































  153 













  153 





  153 






















  153 














  153 




















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
/*
 * hugetlbpage-backed filesystem.  Based on ramfs.
 *
 * Nadia Yvette Chambers, 2002
 *
 * Copyright (C) 2002 Linus Torvalds.
 * License: GPL
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/thread_info.h>
#include <asm/current.h>
#include <linux/falloc.h>
#include <linux/fs.h>
#include <linux/mount.h>
#include <linux/file.h>
#include <linux/kernel.h>
#include <linux/writeback.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/capability.h>
#include <linux/ctype.h>
#include <linux/backing-dev.h>
#include <linux/hugetlb.h>
#include <linux/pagevec.h>
#include <linux/fs_parser.h>
#include <linux/mman.h>
#include <linux/slab.h>
#include <linux/dnotify.h>
#include <linux/statfs.h>
#include <linux/security.h>
#include <linux/magic.h>
#include <linux/migrate.h>
#include <linux/uio.h>

#include <linux/uaccess.h>
#include <linux/sched/mm.h>

#define CREATE_TRACE_POINTS
#include <trace/events/hugetlbfs.h>

static const struct address_space_operations hugetlbfs_aops;
static const struct file_operations hugetlbfs_file_operations;
static const struct inode_operations hugetlbfs_dir_inode_operations;
static const struct inode_operations hugetlbfs_inode_operations;

enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };

struct hugetlbfs_fs_context {
        struct hstate                *hstate;
        unsigned long long        max_size_opt;
        unsigned long long        min_size_opt;
        long                        max_hpages;
        long                        nr_inodes;
        long                        min_hpages;
        enum hugetlbfs_size_type max_val_type;
        enum hugetlbfs_size_type min_val_type;
        kuid_t                        uid;
        kgid_t                        gid;
        umode_t                        mode;
};

int sysctl_hugetlb_shm_group;

enum hugetlb_param {
        Opt_gid,
        Opt_min_size,
        Opt_mode,
        Opt_nr_inodes,
        Opt_pagesize,
        Opt_size,
        Opt_uid,
};

static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
        fsparam_gid   ("gid",                Opt_gid),
        fsparam_string("min_size",        Opt_min_size),
        fsparam_u32oct("mode",                Opt_mode),
        fsparam_string("nr_inodes",        Opt_nr_inodes),
        fsparam_string("pagesize",        Opt_pagesize),
        fsparam_string("size",                Opt_size),
        fsparam_uid   ("uid",                Opt_uid),
        {}
};

/*
 * Mask used when checking the page offset value passed in via system
 * calls.  This value will be converted to a loff_t which is signed.
 * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
 * value.  The extra bit (- 1 in the shift value) is to take the sign
 * bit into account.
 */
#define PGOFF_LOFFT_MAX \
        (((1UL << (PAGE_SHIFT + 1)) - 1) <<  (BITS_PER_LONG - (PAGE_SHIFT + 1)))

static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
{
        struct inode *inode = file_inode(file);
        loff_t len, vma_len;
        int ret;
        struct hstate *h = hstate_file(file);
        vm_flags_t vm_flags;

        /*
         * vma address alignment (but not the pgoff alignment) has
         * already been checked by prepare_hugepage_range.  If you add
         * any error returns here, do so after setting VM_HUGETLB, so
         * is_vm_hugetlb_page tests below unmap_region go the right
         * way when do_mmap unwinds (may be important on powerpc
         * and ia64).
         */
        vm_flags_set(vma, VM_HUGETLB | VM_DONTEXPAND);
        vma->vm_ops = &hugetlb_vm_ops;

        /*
         * page based offset in vm_pgoff could be sufficiently large to
         * overflow a loff_t when converted to byte offset.  This can
         * only happen on architectures where sizeof(loff_t) ==
         * sizeof(unsigned long).  So, only check in those instances.
         */
        if (sizeof(unsigned long) == sizeof(loff_t)) {
                if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
                        return -EINVAL;
        }

        /* must be huge page aligned */
        if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
                return -EINVAL;

        vma_len = (loff_t)(vma->vm_end - vma->vm_start);
        len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
        /* check for overflow */
        if (len < vma_len)
                return -EINVAL;

        inode_lock(inode);
        file_accessed(file);

        ret = -ENOMEM;

        vm_flags = vma->vm_flags;
        /*
         * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip
         * reserving here. Note: only for SHM hugetlbfs file, the inode
         * flag S_PRIVATE is set.
         */
        if (inode->i_flags & S_PRIVATE)
                vm_flags |= VM_NORESERVE;

        if (!hugetlb_reserve_pages(inode,
                                vma->vm_pgoff >> huge_page_order(h),
                                len >> huge_page_shift(h), vma,
                                vm_flags))
                goto out;

        ret = 0;
        if (vma->vm_flags & VM_WRITE && inode->i_size < len)
                i_size_write(inode, len);
out:
        inode_unlock(inode);

        return ret;
}

/*
 * Called under mmap_write_lock(mm).
 */

unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
                            unsigned long len, unsigned long pgoff,
                            unsigned long flags)
{
        unsigned long addr0 = 0;
        struct hstate *h = hstate_file(file);

        if (len & ~huge_page_mask(h))
                return -EINVAL;
        if (flags & MAP_FIXED) {
                if (addr & ~huge_page_mask(h))
                        return -EINVAL;
                if (prepare_hugepage_range(file, addr, len))
                        return -EINVAL;
        }
        if (addr)
                addr0 = ALIGN(addr, huge_page_size(h));

        return mm_get_unmapped_area_vmflags(current->mm, file, addr0, len, pgoff,
                                            flags, 0);
}

/*
 * Someone wants to read @bytes from a HWPOISON hugetlb @folio from @offset.
 * Returns the maximum number of bytes one can read without touching the 1st raw
 * HWPOISON page.
 *
 * The implementation borrows the iteration logic from copy_page_to_iter*.
 */
static size_t adjust_range_hwpoison(struct folio *folio, size_t offset,
                size_t bytes)
{
        struct page *page;
        size_t n = 0;
        size_t res = 0;

        /* First page to start the loop. */
        page = folio_page(folio, offset / PAGE_SIZE);
        offset %= PAGE_SIZE;
        while (1) {
                if (is_raw_hwpoison_page_in_hugepage(page))
                        break;

                /* Safe to read n bytes without touching HWPOISON subpage. */
                n = min(bytes, (size_t)PAGE_SIZE - offset);
                res += n;
                bytes -= n;
                if (!bytes || !n)
                        break;
                offset += n;
                if (offset == PAGE_SIZE) {
                        page = nth_page(page, 1);
                        offset = 0;
                }
        }

        return res;
}

/*
 * Support for read() - Find the page attached to f_mapping and copy out the
 * data. This provides functionality similar to filemap_read().
 */
static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
        struct file *file = iocb->ki_filp;
        struct hstate *h = hstate_file(file);
        struct address_space *mapping = file->f_mapping;
        struct inode *inode = mapping->host;
        unsigned long index = iocb->ki_pos >> huge_page_shift(h);
        unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
        unsigned long end_index;
        loff_t isize;
        ssize_t retval = 0;

        while (iov_iter_count(to)) {
                struct folio *folio;
                size_t nr, copied, want;

                /* nr is the maximum number of bytes to copy from this page */
                nr = huge_page_size(h);
                isize = i_size_read(inode);
                if (!isize)
                        break;
                end_index = (isize - 1) >> huge_page_shift(h);
                if (index > end_index)
                        break;
                if (index == end_index) {
                        nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
                        if (nr <= offset)
                                break;
                }
                nr = nr - offset;

                /* Find the folio */
                folio = filemap_lock_hugetlb_folio(h, mapping, index);
                if (IS_ERR(folio)) {
                        /*
                         * We have a HOLE, zero out the user-buffer for the
                         * length of the hole or request.
                         */
                        copied = iov_iter_zero(nr, to);
                } else {
                        folio_unlock(folio);

                        if (!folio_test_hwpoison(folio))
                                want = nr;
                        else {
                                /*
                                 * Adjust how many bytes safe to read without
                                 * touching the 1st raw HWPOISON page after
                                 * offset.
                                 */
                                want = adjust_range_hwpoison(folio, offset, nr);
                                if (want == 0) {
                                        folio_put(folio);
                                        retval = -EIO;
                                        break;
                                }
                        }

                        /*
                         * We have the folio, copy it to user space buffer.
                         */
                        copied = copy_folio_to_iter(folio, offset, want, to);
                        folio_put(folio);
                }
                offset += copied;
                retval += copied;
                if (copied != nr && iov_iter_count(to)) {
                        if (!retval)
                                retval = -EFAULT;
                        break;
                }
                index += offset >> huge_page_shift(h);
                offset &= ~huge_page_mask(h);
        }
        iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
        return retval;
}

static int hugetlbfs_write_begin(struct file *file,
                        struct address_space *mapping,
                        loff_t pos, unsigned len,
                        struct folio **foliop, void **fsdata)
{
        return -EINVAL;
}

static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
                        loff_t pos, unsigned len, unsigned copied,
                        struct folio *folio, void *fsdata)
{
        BUG();
        return -EINVAL;
}

static void hugetlb_delete_from_page_cache(struct folio *folio)
{
        folio_clear_dirty(folio);
        folio_clear_uptodate(folio);
        filemap_remove_folio(folio);
}

/*
 * Called with i_mmap_rwsem held for inode based vma maps.  This makes
 * sure vma (and vm_mm) will not go away.  We also hold the hugetlb fault
 * mutex for the page in the mapping.  So, we can not race with page being
 * faulted into the vma.
 */
static bool hugetlb_vma_maps_pfn(struct vm_area_struct *vma,
                                unsigned long addr, unsigned long pfn)
{
        pte_t *ptep, pte;

        ptep = hugetlb_walk(vma, addr, huge_page_size(hstate_vma(vma)));
        if (!ptep)
                return false;

        pte = huge_ptep_get(vma->vm_mm, addr, ptep);
        if (huge_pte_none(pte) || !pte_present(pte))
                return false;

        if (pte_pfn(pte) == pfn)
                return true;

        return false;
}

/*
 * Can vma_offset_start/vma_offset_end overflow on 32-bit arches?
 * No, because the interval tree returns us only those vmas
 * which overlap the truncated area starting at pgoff,
 * and no vma on a 32-bit arch can span beyond the 4GB.
 */
static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start)
{
        unsigned long offset = 0;

        if (vma->vm_pgoff < start)
                offset = (start - vma->vm_pgoff) << PAGE_SHIFT;

        return vma->vm_start + offset;
}

static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end)
{
        unsigned long t_end;

        if (!end)
                return vma->vm_end;

        t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start;
        if (t_end > vma->vm_end)
                t_end = vma->vm_end;
        return t_end;
}

/*
 * Called with hugetlb fault mutex held.  Therefore, no more mappings to
 * this folio can be created while executing the routine.
 */
static void hugetlb_unmap_file_folio(struct hstate *h,
                                        struct address_space *mapping,
                                        struct folio *folio, pgoff_t index)
{
        struct rb_root_cached *root = &mapping->i_mmap;
        struct hugetlb_vma_lock *vma_lock;
        unsigned long pfn = folio_pfn(folio);
        struct vm_area_struct *vma;
        unsigned long v_start;
        unsigned long v_end;
        pgoff_t start, end;

        start = index * pages_per_huge_page(h);
        end = (index + 1) * pages_per_huge_page(h);

        i_mmap_lock_write(mapping);
retry:
        vma_lock = NULL;
        vma_interval_tree_foreach(vma, root, start, end - 1) {
                v_start = vma_offset_start(vma, start);
                v_end = vma_offset_end(vma, end);

                if (!hugetlb_vma_maps_pfn(vma, v_start, pfn))
                        continue;

                if (!hugetlb_vma_trylock_write(vma)) {
                        vma_lock = vma->vm_private_data;
                        /*
                         * If we can not get vma lock, we need to drop
                         * immap_sema and take locks in order.  First,
                         * take a ref on the vma_lock structure so that
                         * we can be guaranteed it will not go away when
                         * dropping immap_sema.
                         */
                        kref_get(&vma_lock->refs);
                        break;
                }

                unmap_hugepage_range(vma, v_start, v_end, NULL,
                                     ZAP_FLAG_DROP_MARKER);
                hugetlb_vma_unlock_write(vma);
        }

        i_mmap_unlock_write(mapping);

        if (vma_lock) {
                /*
                 * Wait on vma_lock.  We know it is still valid as we have
                 * a reference.  We must 'open code' vma locking as we do
                 * not know if vma_lock is still attached to vma.
                 */
                down_write(&vma_lock->rw_sema);
                i_mmap_lock_write(mapping);

                vma = vma_lock->vma;
                if (!vma) {
                        /*
                         * If lock is no longer attached to vma, then just
                         * unlock, drop our reference and retry looking for
                         * other vmas.
                         */
                        up_write(&vma_lock->rw_sema);
                        kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
                        goto retry;
                }

                /*
                 * vma_lock is still attached to vma.  Check to see if vma
                 * still maps page and if so, unmap.
                 */
                v_start = vma_offset_start(vma, start);
                v_end = vma_offset_end(vma, end);
                if (hugetlb_vma_maps_pfn(vma, v_start, pfn))
                        unmap_hugepage_range(vma, v_start, v_end, NULL,
                                             ZAP_FLAG_DROP_MARKER);

                kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
                hugetlb_vma_unlock_write(vma);

                goto retry;
        }
}

static void
hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
                      zap_flags_t zap_flags)
{
        struct vm_area_struct *vma;

        /*
         * end == 0 indicates that the entire range after start should be
         * unmapped.  Note, end is exclusive, whereas the interval tree takes
         * an inclusive "last".
         */
        vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
                unsigned long v_start;
                unsigned long v_end;

                if (!hugetlb_vma_trylock_write(vma))
                        continue;

                v_start = vma_offset_start(vma, start);
                v_end = vma_offset_end(vma, end);

                unmap_hugepage_range(vma, v_start, v_end, NULL, zap_flags);

                /*
                 * Note that vma lock only exists for shared/non-private
                 * vmas.  Therefore, lock is not held when calling
                 * unmap_hugepage_range for private vmas.
                 */
                hugetlb_vma_unlock_write(vma);
        }
}

/*
 * Called with hugetlb fault mutex held.
 * Returns true if page was actually removed, false otherwise.
 */
static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
                                        struct address_space *mapping,
                                        struct folio *folio, pgoff_t index,
                                        bool truncate_op)
{
        bool ret = false;

        /*
         * If folio is mapped, it was faulted in after being
         * unmapped in caller.  Unmap (again) while holding
         * the fault mutex.  The mutex will prevent faults
         * until we finish removing the folio.
         */
        if (unlikely(folio_mapped(folio)))
                hugetlb_unmap_file_folio(h, mapping, folio, index);

        folio_lock(folio);
        /*
         * We must remove the folio from page cache before removing
         * the region/ reserve map (hugetlb_unreserve_pages).  In
         * rare out of memory conditions, removal of the region/reserve
         * map could fail.  Correspondingly, the subpool and global
         * reserve usage count can need to be adjusted.
         */
        VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio);
        hugetlb_delete_from_page_cache(folio);
        ret = true;
        if (!truncate_op) {
                if (unlikely(hugetlb_unreserve_pages(inode, index,
                                                        index + 1, 1)))
                        hugetlb_fix_reserve_counts(inode);
        }

        folio_unlock(folio);
        return ret;
}

/*
 * remove_inode_hugepages handles two distinct cases: truncation and hole
 * punch.  There are subtle differences in operation for each case.
 *
 * truncation is indicated by end of range being LLONG_MAX
 *        In this case, we first scan the range and release found pages.
 *        After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
 *        maps and global counts.  Page faults can race with truncation.
 *        During faults, hugetlb_no_page() checks i_size before page allocation,
 *        and again after obtaining page table lock.  It will 'back out'
 *        allocations in the truncated range.
 * hole punch is indicated if end is not LLONG_MAX
 *        In the hole punch case we scan the range and release found pages.
 *        Only when releasing a page is the associated region/reserve map
 *        deleted.  The region/reserve map for ranges without associated
 *        pages are not modified.  Page faults can race with hole punch.
 *        This is indicated if we find a mapped page.
 * Note: If the passed end of range value is beyond the end of file, but
 * not LLONG_MAX this routine still performs a hole punch operation.
 */
static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
                                   loff_t lend)
{
        struct hstate *h = hstate_inode(inode);
        struct address_space *mapping = &inode->i_data;
        const pgoff_t end = lend >> PAGE_SHIFT;
        struct folio_batch fbatch;
        pgoff_t next, index;
        int i, freed = 0;
        bool truncate_op = (lend == LLONG_MAX);

        folio_batch_init(&fbatch);
        next = lstart >> PAGE_SHIFT;
        while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) {
                for (i = 0; i < folio_batch_count(&fbatch); ++i) {
                        struct folio *folio = fbatch.folios[i];
                        u32 hash = 0;

                        index = folio->index >> huge_page_order(h);
                        hash = hugetlb_fault_mutex_hash(mapping, index);
                        mutex_lock(&hugetlb_fault_mutex_table[hash]);

                        /*
                         * Remove folio that was part of folio_batch.
                         */
                        if (remove_inode_single_folio(h, inode, mapping, folio,
                                                        index, truncate_op))
                                freed++;

                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                }
                folio_batch_release(&fbatch);
                cond_resched();
        }

        if (truncate_op)
                (void)hugetlb_unreserve_pages(inode,
                                lstart >> huge_page_shift(h),
                                LONG_MAX, freed);
}

static void hugetlbfs_evict_inode(struct inode *inode)
{
        struct resv_map *resv_map;

        trace_hugetlbfs_evict_inode(inode);
        remove_inode_hugepages(inode, 0, LLONG_MAX);

        /*
         * Get the resv_map from the address space embedded in the inode.
         * This is the address space which points to any resv_map allocated
         * at inode creation time.  If this is a device special inode,
         * i_mapping may not point to the original address space.
         */
        resv_map = (struct resv_map *)(&inode->i_data)->i_private_data;
        /* Only regular and link inodes have associated reserve maps */
        if (resv_map)
                resv_map_release(&resv_map->refs);
        clear_inode(inode);
}

static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
{
        pgoff_t pgoff;
        struct address_space *mapping = inode->i_mapping;
        struct hstate *h = hstate_inode(inode);

        BUG_ON(offset & ~huge_page_mask(h));
        pgoff = offset >> PAGE_SHIFT;

        i_size_write(inode, offset);
        i_mmap_lock_write(mapping);
        if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
                hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
                                      ZAP_FLAG_DROP_MARKER);
        i_mmap_unlock_write(mapping);
        remove_inode_hugepages(inode, offset, LLONG_MAX);
}

static void hugetlbfs_zero_partial_page(struct hstate *h,
                                        struct address_space *mapping,
                                        loff_t start,
                                        loff_t end)
{
        pgoff_t idx = start >> huge_page_shift(h);
        struct folio *folio;

        folio = filemap_lock_hugetlb_folio(h, mapping, idx);
        if (IS_ERR(folio))
                return;

        start = start & ~huge_page_mask(h);
        end = end & ~huge_page_mask(h);
        if (!end)
                end = huge_page_size(h);

        folio_zero_segment(folio, (size_t)start, (size_t)end);

        folio_unlock(folio);
        folio_put(folio);
}

static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
{
        struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
        struct address_space *mapping = inode->i_mapping;
        struct hstate *h = hstate_inode(inode);
        loff_t hpage_size = huge_page_size(h);
        loff_t hole_start, hole_end;

        /*
         * hole_start and hole_end indicate the full pages within the hole.
         */
        hole_start = round_up(offset, hpage_size);
        hole_end = round_down(offset + len, hpage_size);

        inode_lock(inode);

        /* protected by i_rwsem */
        if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
                inode_unlock(inode);
                return -EPERM;
        }

        i_mmap_lock_write(mapping);

        /* If range starts before first full page, zero partial page. */
        if (offset < hole_start)
                hugetlbfs_zero_partial_page(h, mapping,
                                offset, min(offset + len, hole_start));

        /* Unmap users of full pages in the hole. */
        if (hole_end > hole_start) {
                if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
                        hugetlb_vmdelete_list(&mapping->i_mmap,
                                              hole_start >> PAGE_SHIFT,
                                              hole_end >> PAGE_SHIFT, 0);
        }

        /* If range extends beyond last full page, zero partial page. */
        if ((offset + len) > hole_end && (offset + len) > hole_start)
                hugetlbfs_zero_partial_page(h, mapping,
                                hole_end, offset + len);

        i_mmap_unlock_write(mapping);

        /* Remove full pages from the file. */
        if (hole_end > hole_start)
                remove_inode_hugepages(inode, hole_start, hole_end);

        inode_unlock(inode);

        return 0;
}

static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
                                loff_t len)
{
        struct inode *inode = file_inode(file);
        struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
        struct address_space *mapping = inode->i_mapping;
        struct hstate *h = hstate_inode(inode);
        struct vm_area_struct pseudo_vma;
        struct mm_struct *mm = current->mm;
        loff_t hpage_size = huge_page_size(h);
        unsigned long hpage_shift = huge_page_shift(h);
        pgoff_t start, index, end;
        int error;
        u32 hash;

        if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
                return -EOPNOTSUPP;

        if (mode & FALLOC_FL_PUNCH_HOLE) {
                error = hugetlbfs_punch_hole(inode, offset, len);
                goto out_nolock;
        }

        /*
         * Default preallocate case.
         * For this range, start is rounded down and end is rounded up
         * as well as being converted to page offsets.
         */
        start = offset >> hpage_shift;
        end = (offset + len + hpage_size - 1) >> hpage_shift;

        inode_lock(inode);

        /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
        error = inode_newsize_ok(inode, offset + len);
        if (error)
                goto out;

        if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
                error = -EPERM;
                goto out;
        }

        /*
         * Initialize a pseudo vma as this is required by the huge page
         * allocation routines.
         */
        vma_init(&pseudo_vma, mm);
        vm_flags_init(&pseudo_vma, VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
        pseudo_vma.vm_file = file;

        for (index = start; index < end; index++) {
                /*
                 * This is supposed to be the vaddr where the page is being
                 * faulted in, but we have no vaddr here.
                 */
                struct folio *folio;
                unsigned long addr;

                cond_resched();

                /*
                 * fallocate(2) manpage permits EINTR; we may have been
                 * interrupted because we are using up too much memory.
                 */
                if (signal_pending(current)) {
                        error = -EINTR;
                        break;
                }

                /* addr is the offset within the file (zero based) */
                addr = index * hpage_size;

                /* mutex taken here, fault path and hole punch */
                hash = hugetlb_fault_mutex_hash(mapping, index);
                mutex_lock(&hugetlb_fault_mutex_table[hash]);

                /* See if already present in mapping to avoid alloc/free */
                folio = filemap_get_folio(mapping, index << huge_page_order(h));
                if (!IS_ERR(folio)) {
                        folio_put(folio);
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        continue;
                }

                /*
                 * Allocate folio without setting the avoid_reserve argument.
                 * There certainly are no reserves associated with the
                 * pseudo_vma.  However, there could be shared mappings with
                 * reserves for the file at the inode level.  If we fallocate
                 * folios in these areas, we need to consume the reserves
                 * to keep reservation accounting consistent.
                 */
                folio = alloc_hugetlb_folio(&pseudo_vma, addr, false);
                if (IS_ERR(folio)) {
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        error = PTR_ERR(folio);
                        goto out;
                }
                folio_zero_user(folio, addr);
                __folio_mark_uptodate(folio);
                error = hugetlb_add_to_page_cache(folio, mapping, index);
                if (unlikely(error)) {
                        restore_reserve_on_error(h, &pseudo_vma, addr, folio);
                        folio_put(folio);
                        mutex_unlock(&hugetlb_fault_mutex_table[hash]);
                        goto out;
                }

                mutex_unlock(&hugetlb_fault_mutex_table[hash]);

                folio_set_hugetlb_migratable(folio);
                /*
                 * folio_unlock because locked by hugetlb_add_to_page_cache()
                 * folio_put() due to reference from alloc_hugetlb_folio()
                 */
                folio_unlock(folio);
                folio_put(folio);
        }

        if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
                i_size_write(inode, offset + len);
        inode_set_ctime_current(inode);
out:
        inode_unlock(inode);

out_nolock:
        trace_hugetlbfs_fallocate(inode, mode, offset, len, error);
        return error;
}

static int hugetlbfs_setattr(struct mnt_idmap *idmap,
                             struct dentry *dentry, struct iattr *attr)
{
        struct inode *inode = d_inode(dentry);
        struct hstate *h = hstate_inode(inode);
        int error;
        unsigned int ia_valid = attr->ia_valid;
        struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);

        error = setattr_prepare(idmap, dentry, attr);
        if (error)
                return error;

        trace_hugetlbfs_setattr(inode, dentry, attr);

        if (ia_valid & ATTR_SIZE) {
                loff_t oldsize = inode->i_size;
                loff_t newsize = attr->ia_size;

                if (newsize & ~huge_page_mask(h))
                        return -EINVAL;
                /* protected by i_rwsem */
                if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
                    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
                        return -EPERM;
                hugetlb_vmtruncate(inode, newsize);
        }

        setattr_copy(idmap, inode, attr);
        mark_inode_dirty(inode);
        return 0;
}

static struct inode *hugetlbfs_get_root(struct super_block *sb,
                                        struct hugetlbfs_fs_context *ctx)
{
        struct inode *inode;

        inode = new_inode(sb);
        if (inode) {
                inode->i_ino = get_next_ino();
                inode->i_mode = S_IFDIR | ctx->mode;
                inode->i_uid = ctx->uid;
                inode->i_gid = ctx->gid;
                simple_inode_init_ts(inode);
                inode->i_op = &hugetlbfs_dir_inode_operations;
                inode->i_fop = &simple_dir_operations;
                /* directory inodes start off with i_nlink == 2 (for "." entry) */
                inc_nlink(inode);
                lockdep_annotate_inode_mutex_key(inode);
        }
        return inode;
}

/*
 * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
 * be taken from reclaim -- unlike regular filesystems. This needs an
 * annotation because huge_pmd_share() does an allocation under hugetlb's
 * i_mmap_rwsem.
 */
static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;

static struct inode *hugetlbfs_get_inode(struct super_block *sb,
                                        struct mnt_idmap *idmap,
                                        struct inode *dir,
                                        umode_t mode, dev_t dev)
{
        struct inode *inode;
        struct resv_map *resv_map = NULL;

        /*
         * Reserve maps are only needed for inodes that can have associated
         * page allocations.
         */
        if (S_ISREG(mode) || S_ISLNK(mode)) {
                resv_map = resv_map_alloc();
                if (!resv_map)
                        return NULL;
        }

        inode = new_inode(sb);
        if (inode) {
                struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);

                inode->i_ino = get_next_ino();
                inode_init_owner(idmap, inode, dir, mode);
                lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
                                &hugetlbfs_i_mmap_rwsem_key);
                inode->i_mapping->a_ops = &hugetlbfs_aops;
                simple_inode_init_ts(inode);
                inode->i_mapping->i_private_data = resv_map;
                info->seals = F_SEAL_SEAL;
                switch (mode & S_IFMT) {
                default:
                        init_special_inode(inode, mode, dev);
                        break;
                case S_IFREG:
                        inode->i_op = &hugetlbfs_inode_operations;
                        inode->i_fop = &hugetlbfs_file_operations;
                        break;
                case S_IFDIR:
                        inode->i_op = &hugetlbfs_dir_inode_operations;
                        inode->i_fop = &simple_dir_operations;

                        /* directory inodes start off with i_nlink == 2 (for "." entry) */
                        inc_nlink(inode);
                        break;
                case S_IFLNK:
                        inode->i_op = &page_symlink_inode_operations;
                        inode_nohighmem(inode);
                        break;
                }
                lockdep_annotate_inode_mutex_key(inode);
                trace_hugetlbfs_alloc_inode(inode, dir, mode);
        } else {
                if (resv_map)
                        kref_put(&resv_map->refs, resv_map_release);
        }

        return inode;
}

/*
 * File creation. Allocate an inode, and we're done..
 */
static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
                           struct dentry *dentry, umode_t mode, dev_t dev)
{
        struct inode *inode;

        inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, dev);
        if (!inode)
                return -ENOSPC;
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        d_instantiate(dentry, inode);
        dget(dentry);/* Extra count - pin the dentry in core */
        return 0;
}

static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                                      struct dentry *dentry, umode_t mode)
{
        int retval = hugetlbfs_mknod(idmap, dir, dentry,
                                     mode | S_IFDIR, 0);
        if (!retval)
                inc_nlink(dir);
        return ERR_PTR(retval);
}

static int hugetlbfs_create(struct mnt_idmap *idmap,
                            struct inode *dir, struct dentry *dentry,
                            umode_t mode, bool excl)
{
        return hugetlbfs_mknod(idmap, dir, dentry, mode | S_IFREG, 0);
}

static int hugetlbfs_tmpfile(struct mnt_idmap *idmap,
                             struct inode *dir, struct file *file,
                             umode_t mode)
{
        struct inode *inode;

        inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode | S_IFREG, 0);
        if (!inode)
                return -ENOSPC;
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        d_tmpfile(file, inode);
        return finish_open_simple(file, 0);
}

static int hugetlbfs_symlink(struct mnt_idmap *idmap,
                             struct inode *dir, struct dentry *dentry,
                             const char *symname)
{
        const umode_t mode = S_IFLNK|S_IRWXUGO;
        struct inode *inode;
        int error = -ENOSPC;

        inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, 0);
        if (inode) {
                int l = strlen(symname)+1;
                error = page_symlink(inode, symname, l);
                if (!error) {
                        d_instantiate(dentry, inode);
                        dget(dentry);
                } else
                        iput(inode);
        }
        inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));

        return error;
}

#ifdef CONFIG_MIGRATION
static int hugetlbfs_migrate_folio(struct address_space *mapping,
                                struct folio *dst, struct folio *src,
                                enum migrate_mode mode)
{
        int rc;

        rc = migrate_huge_page_move_mapping(mapping, dst, src);
        if (rc != MIGRATEPAGE_SUCCESS)
                return rc;

        if (hugetlb_folio_subpool(src)) {
                hugetlb_set_folio_subpool(dst,
                                        hugetlb_folio_subpool(src));
                hugetlb_set_folio_subpool(src, NULL);
        }

        folio_migrate_flags(dst, src);

        return MIGRATEPAGE_SUCCESS;
}
#else
#define hugetlbfs_migrate_folio NULL
#endif

static int hugetlbfs_error_remove_folio(struct address_space *mapping,
                                struct folio *folio)
{
        return 0;
}

/*
 * Display the mount options in /proc/mounts.
 */
static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
{
        struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
        struct hugepage_subpool *spool = sbinfo->spool;
        unsigned long hpage_size = huge_page_size(sbinfo->hstate);
        unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
        char mod;

        if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
                seq_printf(m, ",uid=%u",
                           from_kuid_munged(&init_user_ns, sbinfo->uid));
        if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
                seq_printf(m, ",gid=%u",
                           from_kgid_munged(&init_user_ns, sbinfo->gid));
        if (sbinfo->mode != 0755)
                seq_printf(m, ",mode=%o", sbinfo->mode);
        if (sbinfo->max_inodes != -1)
                seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);

        hpage_size /= 1024;
        mod = 'K';
        if (hpage_size >= 1024) {
                hpage_size /= 1024;
                mod = 'M';
        }
        seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
        if (spool) {
                if (spool->max_hpages != -1)
                        seq_printf(m, ",size=%llu",
                                   (unsigned long long)spool->max_hpages << hpage_shift);
                if (spool->min_hpages != -1)
                        seq_printf(m, ",min_size=%llu",
                                   (unsigned long long)spool->min_hpages << hpage_shift);
        }
        return 0;
}

static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
{
        struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
        struct hstate *h = hstate_inode(d_inode(dentry));
        u64 id = huge_encode_dev(dentry->d_sb->s_dev);

        buf->f_fsid = u64_to_fsid(id);
        buf->f_type = HUGETLBFS_MAGIC;
        buf->f_bsize = huge_page_size(h);
        if (sbinfo) {
                spin_lock(&sbinfo->stat_lock);
                /* If no limits set, just report 0 or -1 for max/free/used
                 * blocks, like simple_statfs() */
                if (sbinfo->spool) {
                        long free_pages;

                        spin_lock_irq(&sbinfo->spool->lock);
                        buf->f_blocks = sbinfo->spool->max_hpages;
                        free_pages = sbinfo->spool->max_hpages
                                - sbinfo->spool->used_hpages;
                        buf->f_bavail = buf->f_bfree = free_pages;
                        spin_unlock_irq(&sbinfo->spool->lock);
                        buf->f_files = sbinfo->max_inodes;
                        buf->f_ffree = sbinfo->free_inodes;
                }
                spin_unlock(&sbinfo->stat_lock);
        }
        buf->f_namelen = NAME_MAX;
        return 0;
}

static void hugetlbfs_put_super(struct super_block *sb)
{
        struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);

        if (sbi) {
                sb->s_fs_info = NULL;

                if (sbi->spool)
                        hugepage_put_subpool(sbi->spool);

                kfree(sbi);
        }
}

static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
{
        if (sbinfo->free_inodes >= 0) {
                spin_lock(&sbinfo->stat_lock);
                if (unlikely(!sbinfo->free_inodes)) {
                        spin_unlock(&sbinfo->stat_lock);
                        return 0;
                }
                sbinfo->free_inodes--;
                spin_unlock(&sbinfo->stat_lock);
        }

        return 1;
}

static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
{
        if (sbinfo->free_inodes >= 0) {
                spin_lock(&sbinfo->stat_lock);
                sbinfo->free_inodes++;
                spin_unlock(&sbinfo->stat_lock);
        }
}


static struct kmem_cache *hugetlbfs_inode_cachep;

static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
{
        struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
        struct hugetlbfs_inode_info *p;

        if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
                return NULL;
        p = alloc_inode_sb(sb, hugetlbfs_inode_cachep, GFP_KERNEL);
        if (unlikely(!p)) {
                hugetlbfs_inc_free_inodes(sbinfo);
                return NULL;
        }
        return &p->vfs_inode;
}

static void hugetlbfs_free_inode(struct inode *inode)
{
        trace_hugetlbfs_free_inode(inode);
        kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
}

static void hugetlbfs_destroy_inode(struct inode *inode)
{
        hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
}

static const struct address_space_operations hugetlbfs_aops = {
        .write_begin        = hugetlbfs_write_begin,
        .write_end        = hugetlbfs_write_end,
        .dirty_folio        = noop_dirty_folio,
        .migrate_folio  = hugetlbfs_migrate_folio,
        .error_remove_folio        = hugetlbfs_error_remove_folio,
};


static void init_once(void *foo)
{
        struct hugetlbfs_inode_info *ei = foo;

        inode_init_once(&ei->vfs_inode);
}

static const struct file_operations hugetlbfs_file_operations = {
        .read_iter                = hugetlbfs_read_iter,
        .mmap                        = hugetlbfs_file_mmap,
        .fsync                        = noop_fsync,
        .get_unmapped_area        = hugetlb_get_unmapped_area,
        .llseek                        = default_llseek,
        .fallocate                = hugetlbfs_fallocate,
        .fop_flags                = FOP_HUGE_PAGES,
};

static const struct inode_operations hugetlbfs_dir_inode_operations = {
        .create                = hugetlbfs_create,
        .lookup                = simple_lookup,
        .link                = simple_link,
        .unlink                = simple_unlink,
        .symlink        = hugetlbfs_symlink,
        .mkdir                = hugetlbfs_mkdir,
        .rmdir                = simple_rmdir,
        .mknod                = hugetlbfs_mknod,
        .rename                = simple_rename,
        .setattr        = hugetlbfs_setattr,
        .tmpfile        = hugetlbfs_tmpfile,
};

static const struct inode_operations hugetlbfs_inode_operations = {
        .setattr        = hugetlbfs_setattr,
};

static const struct super_operations hugetlbfs_ops = {
        .alloc_inode    = hugetlbfs_alloc_inode,
        .free_inode     = hugetlbfs_free_inode,
        .destroy_inode  = hugetlbfs_destroy_inode,
        .evict_inode        = hugetlbfs_evict_inode,
        .statfs                = hugetlbfs_statfs,
        .put_super        = hugetlbfs_put_super,
        .show_options        = hugetlbfs_show_options,
};

/*
 * Convert size option passed from command line to number of huge pages
 * in the pool specified by hstate.  Size option could be in bytes
 * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
 */
static long
hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
                         enum hugetlbfs_size_type val_type)
{
        if (val_type == NO_SIZE)
                return -1;

        if (val_type == SIZE_PERCENT) {
                size_opt <<= huge_page_shift(h);
                size_opt *= h->max_huge_pages;
                do_div(size_opt, 100);
        }

        size_opt >>= huge_page_shift(h);
        return size_opt;
}

/*
 * Parse one mount parameter.
 */
static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        struct hugetlbfs_fs_context *ctx = fc->fs_private;
        struct fs_parse_result result;
        struct hstate *h;
        char *rest;
        unsigned long ps;
        int opt;

        opt = fs_parse(fc, hugetlb_fs_parameters, param, &result);
        if (opt < 0)
                return opt;

        switch (opt) {
        case Opt_uid:
                ctx->uid = result.uid;
                return 0;

        case Opt_gid:
                ctx->gid = result.gid;
                return 0;

        case Opt_mode:
                ctx->mode = result.uint_32 & 01777U;
                return 0;

        case Opt_size:
                /* memparse() will accept a K/M/G without a digit */
                if (!param->string || !isdigit(param->string[0]))
                        goto bad_val;
                ctx->max_size_opt = memparse(param->string, &rest);
                ctx->max_val_type = SIZE_STD;
                if (*rest == '%')
                        ctx->max_val_type = SIZE_PERCENT;
                return 0;

        case Opt_nr_inodes:
                /* memparse() will accept a K/M/G without a digit */
                if (!param->string || !isdigit(param->string[0]))
                        goto bad_val;
                ctx->nr_inodes = memparse(param->string, &rest);
                return 0;

        case Opt_pagesize:
                ps = memparse(param->string, &rest);
                h = size_to_hstate(ps);
                if (!h) {
                        pr_err("Unsupported page size %lu MB\n", ps / SZ_1M);
                        return -EINVAL;
                }
                ctx->hstate = h;
                return 0;

        case Opt_min_size:
                /* memparse() will accept a K/M/G without a digit */
                if (!param->string || !isdigit(param->string[0]))
                        goto bad_val;
                ctx->min_size_opt = memparse(param->string, &rest);
                ctx->min_val_type = SIZE_STD;
                if (*rest == '%')
                        ctx->min_val_type = SIZE_PERCENT;
                return 0;

        default:
                return -EINVAL;
        }

bad_val:
        return invalfc(fc, "Bad value '%s' for mount option '%s'\n",
                      param->string, param->key);
}

/*
 * Validate the parsed options.
 */
static int hugetlbfs_validate(struct fs_context *fc)
{
        struct hugetlbfs_fs_context *ctx = fc->fs_private;

        /*
         * Use huge page pool size (in hstate) to convert the size
         * options to number of huge pages.  If NO_SIZE, -1 is returned.
         */
        ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
                                                   ctx->max_size_opt,
                                                   ctx->max_val_type);
        ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
                                                   ctx->min_size_opt,
                                                   ctx->min_val_type);

        /*
         * If max_size was specified, then min_size must be smaller
         */
        if (ctx->max_val_type > NO_SIZE &&
            ctx->min_hpages > ctx->max_hpages) {
                pr_err("Minimum size can not be greater than maximum size\n");
                return -EINVAL;
        }

        return 0;
}

static int
hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct hugetlbfs_fs_context *ctx = fc->fs_private;
        struct hugetlbfs_sb_info *sbinfo;

        sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
        if (!sbinfo)
                return -ENOMEM;
        sb->s_fs_info = sbinfo;
        spin_lock_init(&sbinfo->stat_lock);
        sbinfo->hstate                = ctx->hstate;
        sbinfo->max_inodes        = ctx->nr_inodes;
        sbinfo->free_inodes        = ctx->nr_inodes;
        sbinfo->spool                = NULL;
        sbinfo->uid                = ctx->uid;
        sbinfo->gid                = ctx->gid;
        sbinfo->mode                = ctx->mode;

        /*
         * Allocate and initialize subpool if maximum or minimum size is
         * specified.  Any needed reservations (for minimum size) are taken
         * when the subpool is created.
         */
        if (ctx->max_hpages != -1 || ctx->min_hpages != -1) {
                sbinfo->spool = hugepage_new_subpool(ctx->hstate,
                                                     ctx->max_hpages,
                                                     ctx->min_hpages);
                if (!sbinfo->spool)
                        goto out_free;
        }
        sb->s_maxbytes = MAX_LFS_FILESIZE;
        sb->s_blocksize = huge_page_size(ctx->hstate);
        sb->s_blocksize_bits = huge_page_shift(ctx->hstate);
        sb->s_magic = HUGETLBFS_MAGIC;
        sb->s_op = &hugetlbfs_ops;
        sb->s_time_gran = 1;

        /*
         * Due to the special and limited functionality of hugetlbfs, it does
         * not work well as a stacking filesystem.
         */
        sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
        sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx));
        if (!sb->s_root)
                goto out_free;
        return 0;
out_free:
        kfree(sbinfo->spool);
        kfree(sbinfo);
        return -ENOMEM;
}

static int hugetlbfs_get_tree(struct fs_context *fc)
{
        int err = hugetlbfs_validate(fc);
        if (err)
                return err;
        return get_tree_nodev(fc, hugetlbfs_fill_super);
}

static void hugetlbfs_fs_context_free(struct fs_context *fc)
{
        kfree(fc->fs_private);
}

static const struct fs_context_operations hugetlbfs_fs_context_ops = {
        .free                = hugetlbfs_fs_context_free,
        .parse_param        = hugetlbfs_parse_param,
        .get_tree        = hugetlbfs_get_tree,
};

static int hugetlbfs_init_fs_context(struct fs_context *fc)
{
        struct hugetlbfs_fs_context *ctx;

        ctx = kzalloc(sizeof(struct hugetlbfs_fs_context), GFP_KERNEL);
        if (!ctx)
                return -ENOMEM;

        ctx->max_hpages        = -1; /* No limit on size by default */
        ctx->nr_inodes        = -1; /* No limit on number of inodes by default */
        ctx->uid        = current_fsuid();
        ctx->gid        = current_fsgid();
        ctx->mode        = 0755;
        ctx->hstate        = &default_hstate;
        ctx->min_hpages        = -1; /* No default minimum size */
        ctx->max_val_type = NO_SIZE;
        ctx->min_val_type = NO_SIZE;
        fc->fs_private = ctx;
        fc->ops        = &hugetlbfs_fs_context_ops;
        return 0;
}

static struct file_system_type hugetlbfs_fs_type = {
        .name                        = "hugetlbfs",
        .init_fs_context        = hugetlbfs_init_fs_context,
        .parameters                = hugetlb_fs_parameters,
        .kill_sb                = kill_litter_super,
        .fs_flags               = FS_ALLOW_IDMAP,
};

static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];

static int can_do_hugetlb_shm(void)
{
        kgid_t shm_group;
        shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
        return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
}

static int get_hstate_idx(int page_size_log)
{
        struct hstate *h = hstate_sizelog(page_size_log);

        if (!h)
                return -1;
        return hstate_index(h);
}

/*
 * Note that size should be aligned to proper hugepage size in caller side,
 * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
 */
struct file *hugetlb_file_setup(const char *name, size_t size,
                                vm_flags_t acctflag, int creat_flags,
                                int page_size_log)
{
        struct inode *inode;
        struct vfsmount *mnt;
        int hstate_idx;
        struct file *file;

        hstate_idx = get_hstate_idx(page_size_log);
        if (hstate_idx < 0)
                return ERR_PTR(-ENODEV);

        mnt = hugetlbfs_vfsmount[hstate_idx];
        if (!mnt)
                return ERR_PTR(-ENOENT);

        if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
                struct ucounts *ucounts = current_ucounts();

                if (user_shm_lock(size, ucounts)) {
                        pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is obsolete\n",
                                current->comm, current->pid);
                        user_shm_unlock(size, ucounts);
                }
                return ERR_PTR(-EPERM);
        }

        file = ERR_PTR(-ENOSPC);
        /* hugetlbfs_vfsmount[] mounts do not use idmapped mounts.  */
        inode = hugetlbfs_get_inode(mnt->mnt_sb, &nop_mnt_idmap, NULL,
                                    S_IFREG | S_IRWXUGO, 0);
        if (!inode)
                goto out;
        if (creat_flags == HUGETLB_SHMFS_INODE)
                inode->i_flags |= S_PRIVATE;

        inode->i_size = size;
        clear_nlink(inode);

        if (!hugetlb_reserve_pages(inode, 0,
                        size >> huge_page_shift(hstate_inode(inode)), NULL,
                        acctflag))
                file = ERR_PTR(-ENOMEM);
        else
                file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
                                        &hugetlbfs_file_operations);
        if (!IS_ERR(file))
                return file;

        iput(inode);
out:
        return file;
}

static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h)
{
        struct fs_context *fc;
        struct vfsmount *mnt;

        fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT);
        if (IS_ERR(fc)) {
                mnt = ERR_CAST(fc);
        } else {
                struct hugetlbfs_fs_context *ctx = fc->fs_private;
                ctx->hstate = h;
                mnt = fc_mount(fc);
                put_fs_context(fc);
        }
        if (IS_ERR(mnt))
                pr_err("Cannot mount internal hugetlbfs for page size %luK",
                       huge_page_size(h) / SZ_1K);
        return mnt;
}

static int __init init_hugetlbfs_fs(void)
{
        struct vfsmount *mnt;
        struct hstate *h;
        int error;
        int i;

        if (!hugepages_supported()) {
                pr_info("disabling because there are no supported hugepage sizes\n");
                return -ENOTSUPP;
        }

        error = -ENOMEM;
        hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
                                        sizeof(struct hugetlbfs_inode_info),
                                        0, SLAB_ACCOUNT, init_once);
        if (hugetlbfs_inode_cachep == NULL)
                goto out;

        error = register_filesystem(&hugetlbfs_fs_type);
        if (error)
                goto out_free;

        /* default hstate mount is required */
        mnt = mount_one_hugetlbfs(&default_hstate);
        if (IS_ERR(mnt)) {
                error = PTR_ERR(mnt);
                goto out_unreg;
        }
        hugetlbfs_vfsmount[default_hstate_idx] = mnt;

        /* other hstates are optional */
        i = 0;
        for_each_hstate(h) {
                if (i == default_hstate_idx) {
                        i++;
                        continue;
                }

                mnt = mount_one_hugetlbfs(h);
                if (IS_ERR(mnt))
                        hugetlbfs_vfsmount[i] = NULL;
                else
                        hugetlbfs_vfsmount[i] = mnt;
                i++;
        }

        return 0;

 out_unreg:
        (void)unregister_filesystem(&hugetlbfs_fs_type);
 out_free:
        kmem_cache_destroy(hugetlbfs_inode_cachep);
 out:
        return error;
}
fs_initcall(init_hugetlbfs_fs)






































































































































  288 








  288 



  288 







  288 
  288 
  288 








  287 





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 








    1 




































    1 




    1 



































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 






    1 

    1 










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
8073
8074
8075
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087
8088
8089
8090
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104
8105
8106
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
8130
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
8187
8188
8189
8190
8191
8192
8193
8194
8195
8196
8197
8198
8199
8200
8201
8202
8203
8204
8205
8206
8207
8208
8209
8210
8211
8212
8213
8214
8215
8216
8217
8218
8219
8220
8221
8222
8223
8224
8225
8226
8227
8228
8229
8230
8231
8232
8233
8234
8235
8236
8237
8238
8239
8240
8241
8242
8243
8244
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256
8257
8258
8259
8260
8261
8262
8263
8264
8265
8266
8267
8268
8269
8270
8271
8272
8273
8274
8275
8276
8277
8278
8279
8280
8281
8282
8283
8284
8285
8286
8287
8288
8289
8290
8291
8292
8293
8294
8295
8296
8297
8298
8299
8300
8301
8302
8303
8304
8305
8306
8307
8308
8309
8310
8311
8312
8313
8314
8315
8316
8317
8318
8319
8320
8321
8322
8323
8324
8325
8326
8327
8328
8329
8330
8331
8332
8333
8334
8335
8336
8337
8338
8339
8340
8341
8342
8343
8344
8345
8346
8347
8348
8349
8350
8351
8352
8353
8354
8355
8356
8357
8358
8359
8360
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370
8371
8372
8373
8374
8375
8376
8377
8378
8379
8380
8381
8382
8383
8384
8385
8386
8387
8388
8389
8390
8391
8392
8393
8394
8395
8396
8397
8398
8399
8400
8401
8402
8403
8404
8405
8406
8407
8408
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420
8421
8422
8423
8424
8425
8426
8427
8428
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469
8470
8471
8472
8473
8474
8475
8476
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486
8487
8488
8489
8490
8491
8492
8493
8494
8495
8496
8497
8498
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513
8514
8515
8516
8517
8518
8519
8520
8521
8522
8523
8524
8525
8526
8527
8528
8529
8530
8531
8532
8533
8534
8535
8536
8537
8538
8539
8540
8541
8542
8543
8544
8545
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
8585
8586
8587
8588
8589
8590
8591
8592
8593
8594
8595
8596
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609
8610
8611
8612
8613
8614
8615
8616
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
8634
8635
8636
8637
8638
8639
8640
8641
8642
8643
8644
8645
8646
8647
8648
8649
8650
8651
8652
8653
8654
8655
8656
8657
8658
8659
8660
8661
8662
8663
8664
8665
8666
8667
8668
8669
8670
8671
8672
8673
8674
8675
8676
8677
8678
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
8693
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
8715
8716
8717
8718
8719
8720
8721
8722
8723
8724
8725
8726
8727
8728
8729
8730
8731
8732
8733
8734
8735
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751
8752
8753
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765
8766
8767
8768
8769
8770
8771
8772
8773
8774
8775
8776
8777
8778
8779
8780
8781
8782
8783
8784
8785
8786
8787
8788
8789
8790
8791
8792
8793
8794
8795
8796
8797
8798
8799
8800
8801
8802
8803
8804
8805
8806
8807
8808
8809
8810
8811
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
8827
8828
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906
8907
8908
8909
8910
8911
8912
8913
8914
8915
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
8947
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978
8979
8980
8981
8982
8983
8984
8985
8986
8987
8988
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
9007
9008
9009
9010
9011
9012
9013
9014
9015
9016
9017
9018
9019
9020
9021
9022
9023
9024
9025
9026
9027
9028
9029
9030
9031
9032
9033
9034
9035
9036
9037
9038
9039
9040
9041
9042
9043
9044
9045
9046
9047
9048
9049
9050
9051
9052
9053
9054
9055
9056
9057
9058
9059
9060
9061
9062
9063
9064
9065
9066
9067
9068
9069
9070
9071
9072
9073
9074
9075
9076
9077
9078
9079
9080
9081
9082
9083
9084
9085
9086
9087
9088
9089
9090
9091
9092
9093
9094
9095
9096
9097
9098
9099
9100
9101
9102
9103
9104
9105
9106
9107
9108
9109
9110
9111
9112
9113
9114
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
9156
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
9173
9174
9175
9176
9177
9178
9179
9180
9181
9182
9183
9184
9185
9186
9187
9188
9189
9190
9191
9192
9193
9194
9195
9196
9197
9198
9199
9200
9201
9202
9203
9204
9205
9206
9207
9208
9209
9210
9211
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
9226
9227
9228
9229
9230
9231
9232
9233
9234
9235
9236
9237
9238
9239
9240
9241
9242
9243
9244
9245
9246
9247
9248
9249
9250
9251
9252
9253
9254
9255
9256
9257
9258
9259
9260
9261
9262
9263
9264
9265
9266
9267
9268
9269
9270
9271
9272
9273
9274
9275
9276
9277
9278
9279
9280
9281
9282
9283
9284
9285
9286
9287
9288
9289
9290
9291
9292
9293
9294
9295
9296
9297
9298
9299
9300
9301
9302
9303
9304
9305
9306
9307
9308
9309
9310
9311
9312
9313
9314
9315
9316
9317
9318
9319
9320
9321
9322
9323
9324
9325
9326
9327
9328
9329
9330
9331
9332
9333
9334
9335
9336
9337
9338
9339
9340
9341
9342
9343
9344
9345
9346
9347
9348
9349
9350
9351
9352
9353
9354
9355
9356
9357
9358
9359
9360
9361
9362
9363
9364
9365
9366
9367
9368
9369
9370
9371
9372
9373
9374
9375
9376
9377
9378
9379
9380
9381
9382
9383
9384
9385
9386
9387
9388
9389
9390
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
9414
9415
9416
9417
9418
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
9447
9448
9449
9450
9451
9452
9453
9454
9455
9456
9457
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467
9468
9469
9470
9471
9472
9473
9474
9475
9476
9477
9478
9479
9480
9481
9482
9483
9484
9485
9486
9487
9488
9489
9490
9491
9492
9493
9494
9495
9496
9497
9498
9499
9500
9501
9502
9503
9504
9505
9506
9507
9508
9509
9510
9511
9512
9513
9514
9515
9516
9517
9518
9519
9520
9521
9522
9523
9524
9525
9526
9527
9528
9529
9530
9531
9532
9533
9534
9535
9536
9537
9538
9539
9540
9541
9542
9543
9544
9545
9546
9547
9548
9549
9550
9551
9552
9553
9554
9555
9556
9557
9558
9559
9560
9561
9562
9563
9564
9565
9566
9567
9568
9569
9570
9571
9572
9573
9574
9575
9576
9577
9578
9579
9580
9581
9582
9583
9584
9585
9586
9587
9588
9589
9590
9591
9592
9593
9594
9595
9596
9597
9598
9599
9600
9601
9602
9603
9604
9605
9606
9607
9608
9609
9610
9611
9612
9613
9614
9615
9616
9617
9618
9619
9620
9621
9622
9623
9624
9625
9626
9627
9628
9629
9630
9631
9632
9633
9634
9635
9636
9637
9638
9639
9640
9641
9642
9643
9644
9645
9646
9647
9648
9649
9650
9651
9652
9653
9654
9655
9656
9657
9658
9659
9660
9661
9662
9663
9664
9665
9666
9667
9668
9669
9670
9671
9672
9673
9674
9675
9676
9677
9678
9679
9680
9681
9682
9683
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701
9702
9703
9704
9705
9706
9707
9708
9709
9710
9711
9712
9713
9714
9715
9716
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727
9728
9729
9730
9731
9732
9733
9734
9735
9736
9737
9738
9739
9740
9741
9742
9743
9744
9745
9746
9747
9748
9749
9750
9751
9752
9753
9754
9755
9756
9757
9758
9759
9760
9761
9762
9763
9764
9765
9766
9767
9768
9769
9770
9771
9772
9773
9774
9775
9776
9777
9778
9779
9780
9781
9782
9783
9784
9785
9786
9787
9788
9789
9790
9791
9792
9793
9794
9795
9796
9797
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814
9815
9816
9817
9818
9819
9820
9821
9822
9823
9824
9825
9826
9827
9828
9829
9830
9831
9832
9833
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845
9846
9847
9848
9849
9850
9851
9852
9853
9854
9855
9856
9857
9858
9859
9860
9861
9862
9863
9864
9865
9866
9867
9868
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902
9903
9904
9905
9906
9907
9908
9909
9910
9911
9912
9913
9914
9915
9916
9917
9918
9919
9920
9921
9922
9923
9924
9925
9926
9927
9928
9929
9930
9931
9932
9933
9934
9935
9936
9937
9938
9939
9940
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952
9953
9954
9955
9956
9957
9958
9959
9960
9961
9962
9963
9964
9965
9966
9967
9968
9969
9970
9971
9972
9973
9974
9975
9976
9977
9978
9979
9980
9981
9982
9983
9984
9985
9986
9987
9988
9989
9990
9991
9992
9993
9994
9995
9996
9997
9998
9999
10000
10001
10002
10003
10004
10005
10006
10007
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
10080
10081
10082
10083
10084
10085
10086
10087
10088
10089
10090
10091
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102
10103
10104
10105
10106
10107
10108
10109
10110
10111
10112
10113
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141
10142
10143
10144
10145
10146
10147
10148
10149
10150
10151
10152
10153
10154
10155
10156
10157
10158
10159
10160
10161
10162
10163
10164
10165
10166
10167
10168
10169
10170
10171
10172
10173
10174
10175
10176
10177
10178
10179
10180
10181
10182
10183
10184
10185
10186
10187
10188
10189
10190
10191
10192
10193
10194
10195
10196
10197
10198
10199
10200
10201
10202
10203
10204
10205
10206
10207
10208
10209
10210
10211
10212
10213
10214
10215
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
10232
10233
10234
10235
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245
10246
10247
10248
10249
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
10295
10296
10297
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
10438
10439
10440
10441
10442
10443
10444
10445
10446
10447
10448
10449
10450
10451
10452
10453
10454
10455
10456
10457
10458
10459
10460
10461
10462
10463
10464
10465
10466
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477
10478
10479
10480
10481
10482
10483
10484
10485
10486
10487
10488
10489
10490
10491
10492
10493
10494
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
10526
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
10547
10548
10549
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
10641
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
10713
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
11027
11028
11029
11030
11031
11032
11033
11034
11035
11036
11037
11038
11039
11040
11041
11042
11043
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054
11055
11056
11057
11058
11059
11060
11061
11062
11063
11064
11065
11066
11067
11068
11069
11070
11071
11072
11073
11074
11075
11076
11077
11078
11079
11080
11081
11082
11083
11084
11085
11086
11087
11088
11089
11090
11091
11092
11093
11094
11095
11096
11097
11098
11099
11100
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119
11120
11121
11122
11123
11124
11125
11126
11127
11128
11129
11130
11131
11132
11133
11134
11135
11136
11137
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
11444
11445
11446
11447
11448
11449
11450
11451
11452
11453
11454
11455
11456
11457
11458
11459
11460
11461
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
11591
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
11691
11692
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
11736
11737
11738
11739
11740
11741
11742
11743
11744
11745
11746
11747
11748
11749
11750
11751
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
11778
11779
11780
11781
11782
11783
11784
11785
11786
11787
11788
11789
11790
11791
11792
11793
11794
11795
11796
11797
11798
11799
11800
11801
11802
11803
11804
11805
11806
11807
11808
11809
11810
11811
11812
11813
11814
11815
11816
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828
11829
11830
11831
11832
11833
11834
11835
11836
11837
11838
11839
11840
11841
11842
11843
11844
11845
11846
11847
11848
11849
11850
11851
11852
11853
11854
11855
11856
11857
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867
11868
11869
11870
11871
11872
11873
11874
11875
11876
11877
11878
11879
11880
11881
11882
11883
11884
11885
11886
11887
11888
11889
11890
11891
11892
11893
11894
11895
11896
11897
11898
11899
11900
11901
11902
11903
11904
11905
11906
11907
11908
11909
11910
11911
11912
11913
11914
11915
11916
11917
11918
11919
11920
11921
11922
11923
11924
11925
11926
11927
11928
11929
11930
11931
11932
11933
11934
11935
11936
11937
11938
11939
11940
11941
11942
11943
11944
11945
11946
11947
11948
11949
11950
11951
11952
11953
11954
11955
11956
11957
11958
11959
11960
11961
11962
11963
11964
11965
11966
11967
11968
11969
11970
11971
11972
11973
11974
11975
11976
11977
11978
11979
11980
11981
11982
11983
11984
11985
11986
11987
11988
11989
11990
11991
11992
11993
11994
11995
11996
11997
11998
11999
12000
12001
12002
12003
12004
12005
12006
12007
12008
12009
12010
12011
12012
12013
12014
12015
12016
12017
12018
12019
12020
12021
12022
12023
12024
12025
12026
12027
12028
12029
12030
12031
12032
12033
12034
12035
12036
12037
12038
12039
12040
12041
12042
12043
12044
12045
12046
12047
12048
12049
12050
12051
12052
12053
12054
12055
12056
12057
12058
12059
12060
12061
12062
12063
12064
12065
12066
12067
12068
12069
12070
12071
12072
12073
12074
12075
12076
12077
12078
12079
12080
12081
12082
12083
12084
12085
12086
12087
12088
12089
12090
12091
12092
12093
12094
12095
12096
12097
12098
12099
12100
12101
12102
12103
12104
12105
12106
12107
12108
12109
12110
12111
12112
12113
12114
12115
12116
12117
12118
12119
12120
12121
12122
12123
12124
12125
12126
12127
12128
12129
12130
12131
12132
12133
12134
12135
12136
12137
12138
12139
12140
12141
12142
12143
12144
12145
12146
12147
12148
12149
12150
12151
12152
12153
12154
12155
12156
12157
12158
12159
12160
12161
12162
12163
12164
12165
12166
12167
12168
12169
12170
12171
12172
12173
12174
12175
12176
12177
12178
12179
12180
12181
12182
12183
12184
12185
12186
12187
12188
12189
12190
12191
12192
12193
12194
12195
12196
12197
12198
12199
12200
12201
12202
12203
12204
12205
12206
12207
12208
12209
12210
12211
12212
12213
12214
12215
12216
12217
12218
12219
12220
12221
12222
12223
12224
12225
12226
12227
12228
12229
12230
12231
12232
12233
12234
12235
12236
12237
12238
12239
12240
12241
12242
12243
12244
12245
12246
12247
12248
12249
12250
12251
12252
12253
12254
12255
12256
12257
12258
12259
12260
12261
12262
12263
12264
12265
12266
12267
12268
12269
12270
12271
12272
12273
12274
12275
12276
12277
12278
12279
12280
12281
12282
12283
12284
12285
12286
12287
12288
12289
12290
12291
12292
12293
12294
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Linux Socket Filter - Kernel level socket filtering
 *
 * Based on the design of the Berkeley Packet Filter. The new
 * internal format has been designed by PLUMgrid:
 *
 *        Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com
 *
 * Authors:
 *
 *        Jay Schulist <jschlst@samba.org>
 *        Alexei Starovoitov <ast@plumgrid.com>
 *        Daniel Borkmann <dborkman@redhat.com>
 *
 * Andi Kleen - Fix a few bad bugs and races.
 * Kris Katterjohn - Added many additional checks in bpf_check_classic()
 */

#include <linux/atomic.h>
#include <linux/bpf_verifier.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/fcntl.h>
#include <linux/socket.h>
#include <linux/sock_diag.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_packet.h>
#include <linux/if_arp.h>
#include <linux/gfp.h>
#include <net/inet_common.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/netlink.h>
#include <linux/skbuff.h>
#include <linux/skmsg.h>
#include <net/sock.h>
#include <net/flow_dissector.h>
#include <linux/errno.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
#include <linux/unaligned.h>
#include <linux/filter.h>
#include <linux/ratelimit.h>
#include <linux/seccomp.h>
#include <linux/if_vlan.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <net/sch_generic.h>
#include <net/cls_cgroup.h>
#include <net/dst_metadata.h>
#include <net/dst.h>
#include <net/sock_reuseport.h>
#include <net/busy_poll.h>
#include <net/tcp.h>
#include <net/xfrm.h>
#include <net/udp.h>
#include <linux/bpf_trace.h>
#include <net/xdp_sock.h>
#include <linux/inetdevice.h>
#include <net/inet_hashtables.h>
#include <net/inet6_hashtables.h>
#include <net/ip_fib.h>
#include <net/nexthop.h>
#include <net/flow.h>
#include <net/arp.h>
#include <net/ipv6.h>
#include <net/net_namespace.h>
#include <linux/seg6_local.h>
#include <net/seg6.h>
#include <net/seg6_local.h>
#include <net/lwtunnel.h>
#include <net/ipv6_stubs.h>
#include <net/bpf_sk_storage.h>
#include <net/transp_v6.h>
#include <linux/btf_ids.h>
#include <net/tls.h>
#include <net/xdp.h>
#include <net/mptcp.h>
#include <net/netfilter/nf_conntrack_bpf.h>
#include <net/netkit.h>
#include <linux/un.h>
#include <net/xdp_sock_drv.h>
#include <net/inet_dscp.h>

#include "dev.h"

/* Keep the struct bpf_fib_lookup small so that it fits into a cacheline */
static_assert(sizeof(struct bpf_fib_lookup) == 64, "struct bpf_fib_lookup size check");

static const struct bpf_func_proto *
bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);

int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len)
{
        if (in_compat_syscall()) {
                struct compat_sock_fprog f32;

                if (len != sizeof(f32))
                        return -EINVAL;
                if (copy_from_sockptr(&f32, src, sizeof(f32)))
                        return -EFAULT;
                memset(dst, 0, sizeof(*dst));
                dst->len = f32.len;
                dst->filter = compat_ptr(f32.filter);
        } else {
                if (len != sizeof(*dst))
                        return -EINVAL;
                if (copy_from_sockptr(dst, src, sizeof(*dst)))
                        return -EFAULT;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(copy_bpf_fprog_from_user);

/**
 *        sk_filter_trim_cap - run a packet through a socket filter
 *        @sk: sock associated with &sk_buff
 *        @skb: buffer to filter
 *        @cap: limit on how short the eBPF program may trim the packet
 *
 * Run the eBPF program and then cut skb->data to correct size returned by
 * the program. If pkt_len is 0 we toss packet. If skb->len is smaller
 * than pkt_len we keep whole skb->data. This is the socket level
 * wrapper to bpf_prog_run. It returns 0 if the packet should
 * be accepted or -EPERM if the packet should be tossed.
 *
 */
int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap)
{
        int err;
        struct sk_filter *filter;

        /*
         * If the skb was allocated from pfmemalloc reserves, only
         * allow SOCK_MEMALLOC sockets to use it as this socket is
         * helping free memory
         */
        if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC)) {
                NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
                return -ENOMEM;
        }
        err = BPF_CGROUP_RUN_PROG_INET_INGRESS(sk, skb);
        if (err)
                return err;

        err = security_sock_rcv_skb(sk, skb);
        if (err)
                return err;

        rcu_read_lock();
        filter = rcu_dereference(sk->sk_filter);
        if (filter) {
                struct sock *save_sk = skb->sk;
                unsigned int pkt_len;

                skb->sk = sk;
                pkt_len = bpf_prog_run_save_cb(filter->prog, skb);
                skb->sk = save_sk;
                err = pkt_len ? pskb_trim(skb, max(cap, pkt_len)) : -EPERM;
        }
        rcu_read_unlock();

        return err;
}
EXPORT_SYMBOL(sk_filter_trim_cap);

BPF_CALL_1(bpf_skb_get_pay_offset, struct sk_buff *, skb)
{
        return skb_get_poff(skb);
}

BPF_CALL_3(bpf_skb_get_nlattr, struct sk_buff *, skb, u32, a, u32, x)
{
        struct nlattr *nla;

        if (skb_is_nonlinear(skb))
                return 0;

        if (skb->len < sizeof(struct nlattr))
                return 0;

        if (a > skb->len - sizeof(struct nlattr))
                return 0;

        nla = nla_find((struct nlattr *) &skb->data[a], skb->len - a, x);
        if (nla)
                return (void *) nla - (void *) skb->data;

        return 0;
}

BPF_CALL_3(bpf_skb_get_nlattr_nest, struct sk_buff *, skb, u32, a, u32, x)
{
        struct nlattr *nla;

        if (skb_is_nonlinear(skb))
                return 0;

        if (skb->len < sizeof(struct nlattr))
                return 0;

        if (a > skb->len - sizeof(struct nlattr))
                return 0;

        nla = (struct nlattr *) &skb->data[a];
        if (!nla_ok(nla, skb->len - a))
                return 0;

        nla = nla_find_nested(nla, x);
        if (nla)
                return (void *) nla - (void *) skb->data;

        return 0;
}

static int bpf_skb_load_helper_convert_offset(const struct sk_buff *skb, int offset)
{
        if (likely(offset >= 0))
                return offset;

        if (offset >= SKF_NET_OFF)
                return offset - SKF_NET_OFF + skb_network_offset(skb);

        if (offset >= SKF_LL_OFF && skb_mac_header_was_set(skb))
                return offset - SKF_LL_OFF + skb_mac_offset(skb);

        return INT_MIN;
}

BPF_CALL_4(bpf_skb_load_helper_8, const struct sk_buff *, skb, const void *,
           data, int, headlen, int, offset)
{
        u8 tmp;
        const int len = sizeof(tmp);

        offset = bpf_skb_load_helper_convert_offset(skb, offset);
        if (offset == INT_MIN)
                return -EFAULT;

        if (headlen - offset >= len)
                return *(u8 *)(data + offset);
        if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
                return tmp;
        else
                return -EFAULT;
}

BPF_CALL_2(bpf_skb_load_helper_8_no_cache, const struct sk_buff *, skb,
           int, offset)
{
        return ____bpf_skb_load_helper_8(skb, skb->data, skb->len - skb->data_len,
                                         offset);
}

BPF_CALL_4(bpf_skb_load_helper_16, const struct sk_buff *, skb, const void *,
           data, int, headlen, int, offset)
{
        __be16 tmp;
        const int len = sizeof(tmp);

        offset = bpf_skb_load_helper_convert_offset(skb, offset);
        if (offset == INT_MIN)
                return -EFAULT;

        if (headlen - offset >= len)
                return get_unaligned_be16(data + offset);
        if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
                return be16_to_cpu(tmp);
        else
                return -EFAULT;
}

BPF_CALL_2(bpf_skb_load_helper_16_no_cache, const struct sk_buff *, skb,
           int, offset)
{
        return ____bpf_skb_load_helper_16(skb, skb->data, skb->len - skb->data_len,
                                          offset);
}

BPF_CALL_4(bpf_skb_load_helper_32, const struct sk_buff *, skb, const void *,
           data, int, headlen, int, offset)
{
        __be32 tmp;
        const int len = sizeof(tmp);

        offset = bpf_skb_load_helper_convert_offset(skb, offset);
        if (offset == INT_MIN)
                return -EFAULT;

        if (headlen - offset >= len)
                return get_unaligned_be32(data + offset);
        if (!skb_copy_bits(skb, offset, &tmp, sizeof(tmp)))
                return be32_to_cpu(tmp);
        else
                return -EFAULT;
}

BPF_CALL_2(bpf_skb_load_helper_32_no_cache, const struct sk_buff *, skb,
           int, offset)
{
        return ____bpf_skb_load_helper_32(skb, skb->data, skb->len - skb->data_len,
                                          offset);
}

static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg,
                              struct bpf_insn *insn_buf)
{
        struct bpf_insn *insn = insn_buf;

        switch (skb_field) {
        case SKF_AD_MARK:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4);

                *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
                                      offsetof(struct sk_buff, mark));
                break;

        case SKF_AD_PKTTYPE:
                *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET);
                *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX);
#ifdef __BIG_ENDIAN_BITFIELD
                *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5);
#endif
                break;

        case SKF_AD_QUEUE:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, queue_mapping) != 2);

                *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
                                      offsetof(struct sk_buff, queue_mapping));
                break;

        case SKF_AD_VLAN_TAG:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_tci) != 2);

                /* dst_reg = *(u16 *) (src_reg + offsetof(vlan_tci)) */
                *insn++ = BPF_LDX_MEM(BPF_H, dst_reg, src_reg,
                                      offsetof(struct sk_buff, vlan_tci));
                break;
        case SKF_AD_VLAN_TAG_PRESENT:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_all) != 4);
                *insn++ = BPF_LDX_MEM(BPF_W, dst_reg, src_reg,
                                      offsetof(struct sk_buff, vlan_all));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, dst_reg, 0, 1);
                *insn++ = BPF_ALU32_IMM(BPF_MOV, dst_reg, 1);
                break;
        }

        return insn - insn_buf;
}

static bool convert_bpf_extensions(struct sock_filter *fp,
                                   struct bpf_insn **insnp)
{
        struct bpf_insn *insn = *insnp;
        u32 cnt;

        switch (fp->k) {
        case SKF_AD_OFF + SKF_AD_PROTOCOL:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, protocol) != 2);

                /* A = *(u16 *) (CTX + offsetof(protocol)) */
                *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
                                      offsetof(struct sk_buff, protocol));
                /* A = ntohs(A) [emitting a nop or swap16] */
                *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
                break;

        case SKF_AD_OFF + SKF_AD_PKTTYPE:
                cnt = convert_skb_access(SKF_AD_PKTTYPE, BPF_REG_A, BPF_REG_CTX, insn);
                insn += cnt - 1;
                break;

        case SKF_AD_OFF + SKF_AD_IFINDEX:
        case SKF_AD_OFF + SKF_AD_HATYPE:
                BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4);
                BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
                                      BPF_REG_TMP, BPF_REG_CTX,
                                      offsetof(struct sk_buff, dev));
                /* if (tmp != 0) goto pc + 1 */
                *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_TMP, 0, 1);
                *insn++ = BPF_EXIT_INSN();
                if (fp->k == SKF_AD_OFF + SKF_AD_IFINDEX)
                        *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_TMP,
                                            offsetof(struct net_device, ifindex));
                else
                        *insn = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_TMP,
                                            offsetof(struct net_device, type));
                break;

        case SKF_AD_OFF + SKF_AD_MARK:
                cnt = convert_skb_access(SKF_AD_MARK, BPF_REG_A, BPF_REG_CTX, insn);
                insn += cnt - 1;
                break;

        case SKF_AD_OFF + SKF_AD_RXHASH:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4);

                *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX,
                                    offsetof(struct sk_buff, hash));
                break;

        case SKF_AD_OFF + SKF_AD_QUEUE:
                cnt = convert_skb_access(SKF_AD_QUEUE, BPF_REG_A, BPF_REG_CTX, insn);
                insn += cnt - 1;
                break;

        case SKF_AD_OFF + SKF_AD_VLAN_TAG:
                cnt = convert_skb_access(SKF_AD_VLAN_TAG,
                                         BPF_REG_A, BPF_REG_CTX, insn);
                insn += cnt - 1;
                break;

        case SKF_AD_OFF + SKF_AD_VLAN_TAG_PRESENT:
                cnt = convert_skb_access(SKF_AD_VLAN_TAG_PRESENT,
                                         BPF_REG_A, BPF_REG_CTX, insn);
                insn += cnt - 1;
                break;

        case SKF_AD_OFF + SKF_AD_VLAN_TPID:
                BUILD_BUG_ON(sizeof_field(struct sk_buff, vlan_proto) != 2);

                /* A = *(u16 *) (CTX + offsetof(vlan_proto)) */
                *insn++ = BPF_LDX_MEM(BPF_H, BPF_REG_A, BPF_REG_CTX,
                                      offsetof(struct sk_buff, vlan_proto));
                /* A = ntohs(A) [emitting a nop or swap16] */
                *insn = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, 16);
                break;

        case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
        case SKF_AD_OFF + SKF_AD_NLATTR:
        case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
        case SKF_AD_OFF + SKF_AD_CPU:
        case SKF_AD_OFF + SKF_AD_RANDOM:
                /* arg1 = CTX */
                *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
                /* arg2 = A */
                *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_A);
                /* arg3 = X */
                *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_X);
                /* Emit call(arg1=CTX, arg2=A, arg3=X) */
                switch (fp->k) {
                case SKF_AD_OFF + SKF_AD_PAY_OFFSET:
                        *insn = BPF_EMIT_CALL(bpf_skb_get_pay_offset);
                        break;
                case SKF_AD_OFF + SKF_AD_NLATTR:
                        *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr);
                        break;
                case SKF_AD_OFF + SKF_AD_NLATTR_NEST:
                        *insn = BPF_EMIT_CALL(bpf_skb_get_nlattr_nest);
                        break;
                case SKF_AD_OFF + SKF_AD_CPU:
                        *insn = BPF_EMIT_CALL(bpf_get_raw_cpu_id);
                        break;
                case SKF_AD_OFF + SKF_AD_RANDOM:
                        *insn = BPF_EMIT_CALL(bpf_user_rnd_u32);
                        bpf_user_rnd_init_once();
                        break;
                }
                break;

        case SKF_AD_OFF + SKF_AD_ALU_XOR_X:
                /* A ^= X */
                *insn = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_X);
                break;

        default:
                /* This is just a dummy call to avoid letting the compiler
                 * evict __bpf_call_base() as an optimization. Placed here
                 * where no-one bothers.
                 */
                BUG_ON(__bpf_call_base(0, 0, 0, 0, 0) != 0);
                return false;
        }

        *insnp = insn;
        return true;
}

static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp)
{
        const bool unaligned_ok = IS_BUILTIN(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS);
        int size = bpf_size_to_bytes(BPF_SIZE(fp->code));
        bool endian = BPF_SIZE(fp->code) == BPF_H ||
                      BPF_SIZE(fp->code) == BPF_W;
        bool indirect = BPF_MODE(fp->code) == BPF_IND;
        const int ip_align = NET_IP_ALIGN;
        struct bpf_insn *insn = *insnp;
        int offset = fp->k;

        if (!indirect &&
            ((unaligned_ok && offset >= 0) ||
             (!unaligned_ok && offset >= 0 &&
              offset + ip_align >= 0 &&
              offset + ip_align % size == 0))) {
                bool ldx_off_ok = offset <= S16_MAX;

                *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
                if (offset)
                        *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);
                *insn++ = BPF_JMP_IMM(BPF_JSLT, BPF_REG_TMP,
                                      size, 2 + endian + (!ldx_off_ok * 2));
                if (ldx_off_ok) {
                        *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
                                              BPF_REG_D, offset);
                } else {
                        *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_D);
                        *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_TMP, offset);
                        *insn++ = BPF_LDX_MEM(BPF_SIZE(fp->code), BPF_REG_A,
                                              BPF_REG_TMP, 0);
                }
                if (endian)
                        *insn++ = BPF_ENDIAN(BPF_FROM_BE, BPF_REG_A, size * 8);
                *insn++ = BPF_JMP_A(8);
        }

        *insn++ = BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX);
        *insn++ = BPF_MOV64_REG(BPF_REG_ARG2, BPF_REG_D);
        *insn++ = BPF_MOV64_REG(BPF_REG_ARG3, BPF_REG_H);
        if (!indirect) {
                *insn++ = BPF_MOV64_IMM(BPF_REG_ARG4, offset);
        } else {
                *insn++ = BPF_MOV64_REG(BPF_REG_ARG4, BPF_REG_X);
                if (fp->k)
                        *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG4, offset);
        }

        switch (BPF_SIZE(fp->code)) {
        case BPF_B:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8);
                break;
        case BPF_H:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16);
                break;
        case BPF_W:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32);
                break;
        default:
                return false;
        }

        *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_A, 0, 2);
        *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
        *insn   = BPF_EXIT_INSN();

        *insnp = insn;
        return true;
}

/**
 *        bpf_convert_filter - convert filter program
 *        @prog: the user passed filter program
 *        @len: the length of the user passed filter program
 *        @new_prog: allocated 'struct bpf_prog' or NULL
 *        @new_len: pointer to store length of converted program
 *        @seen_ld_abs: bool whether we've seen ld_abs/ind
 *
 * Remap 'sock_filter' style classic BPF (cBPF) instruction set to 'bpf_insn'
 * style extended BPF (eBPF).
 * Conversion workflow:
 *
 * 1) First pass for calculating the new program length:
 *   bpf_convert_filter(old_prog, old_len, NULL, &new_len, &seen_ld_abs)
 *
 * 2) 2nd pass to remap in two passes: 1st pass finds new
 *    jump offsets, 2nd pass remapping:
 *   bpf_convert_filter(old_prog, old_len, new_prog, &new_len, &seen_ld_abs)
 */
static int bpf_convert_filter(struct sock_filter *prog, int len,
                              struct bpf_prog *new_prog, int *new_len,
                              bool *seen_ld_abs)
{
        int new_flen = 0, pass = 0, target, i, stack_off;
        struct bpf_insn *new_insn, *first_insn = NULL;
        struct sock_filter *fp;
        int *addrs = NULL;
        u8 bpf_src;

        BUILD_BUG_ON(BPF_MEMWORDS * sizeof(u32) > MAX_BPF_STACK);
        BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);

        if (len <= 0 || len > BPF_MAXINSNS)
                return -EINVAL;

        if (new_prog) {
                first_insn = new_prog->insnsi;
                addrs = kcalloc(len, sizeof(*addrs),
                                GFP_KERNEL | __GFP_NOWARN);
                if (!addrs)
                        return -ENOMEM;
        }

do_pass:
        new_insn = first_insn;
        fp = prog;

        /* Classic BPF related prologue emission. */
        if (new_prog) {
                /* Classic BPF expects A and X to be reset first. These need
                 * to be guaranteed to be the first two instructions.
                 */
                *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
                *new_insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_X, BPF_REG_X);

                /* All programs must keep CTX in callee saved BPF_REG_CTX.
                 * In eBPF case it's done by the compiler, here we need to
                 * do this ourself. Initial CTX is present in BPF_REG_ARG1.
                 */
                *new_insn++ = BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1);
                if (*seen_ld_abs) {
                        /* For packet access in classic BPF, cache skb->data
                         * in callee-saved BPF R8 and skb->len - skb->data_len
                         * (headlen) in BPF R9. Since classic BPF is read-only
                         * on CTX, we only need to cache it once.
                         */
                        *new_insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
                                                  BPF_REG_D, BPF_REG_CTX,
                                                  offsetof(struct sk_buff, data));
                        *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_H, BPF_REG_CTX,
                                                  offsetof(struct sk_buff, len));
                        *new_insn++ = BPF_LDX_MEM(BPF_W, BPF_REG_TMP, BPF_REG_CTX,
                                                  offsetof(struct sk_buff, data_len));
                        *new_insn++ = BPF_ALU32_REG(BPF_SUB, BPF_REG_H, BPF_REG_TMP);
                }
        } else {
                new_insn += 3;
        }

        for (i = 0; i < len; fp++, i++) {
                struct bpf_insn tmp_insns[32] = { };
                struct bpf_insn *insn = tmp_insns;

                if (addrs)
                        addrs[i] = new_insn - first_insn;

                switch (fp->code) {
                /* All arithmetic insns and skb loads map as-is. */
                case BPF_ALU | BPF_ADD | BPF_X:
                case BPF_ALU | BPF_ADD | BPF_K:
                case BPF_ALU | BPF_SUB | BPF_X:
                case BPF_ALU | BPF_SUB | BPF_K:
                case BPF_ALU | BPF_AND | BPF_X:
                case BPF_ALU | BPF_AND | BPF_K:
                case BPF_ALU | BPF_OR | BPF_X:
                case BPF_ALU | BPF_OR | BPF_K:
                case BPF_ALU | BPF_LSH | BPF_X:
                case BPF_ALU | BPF_LSH | BPF_K:
                case BPF_ALU | BPF_RSH | BPF_X:
                case BPF_ALU | BPF_RSH | BPF_K:
                case BPF_ALU | BPF_XOR | BPF_X:
                case BPF_ALU | BPF_XOR | BPF_K:
                case BPF_ALU | BPF_MUL | BPF_X:
                case BPF_ALU | BPF_MUL | BPF_K:
                case BPF_ALU | BPF_DIV | BPF_X:
                case BPF_ALU | BPF_DIV | BPF_K:
                case BPF_ALU | BPF_MOD | BPF_X:
                case BPF_ALU | BPF_MOD | BPF_K:
                case BPF_ALU | BPF_NEG:
                case BPF_LD | BPF_ABS | BPF_W:
                case BPF_LD | BPF_ABS | BPF_H:
                case BPF_LD | BPF_ABS | BPF_B:
                case BPF_LD | BPF_IND | BPF_W:
                case BPF_LD | BPF_IND | BPF_H:
                case BPF_LD | BPF_IND | BPF_B:
                        /* Check for overloaded BPF extension and
                         * directly convert it if found, otherwise
                         * just move on with mapping.
                         */
                        if (BPF_CLASS(fp->code) == BPF_LD &&
                            BPF_MODE(fp->code) == BPF_ABS &&
                            convert_bpf_extensions(fp, &insn))
                                break;
                        if (BPF_CLASS(fp->code) == BPF_LD &&
                            convert_bpf_ld_abs(fp, &insn)) {
                                *seen_ld_abs = true;
                                break;
                        }

                        if (fp->code == (BPF_ALU | BPF_DIV | BPF_X) ||
                            fp->code == (BPF_ALU | BPF_MOD | BPF_X)) {
                                *insn++ = BPF_MOV32_REG(BPF_REG_X, BPF_REG_X);
                                /* Error with exception code on div/mod by 0.
                                 * For cBPF programs, this was always return 0.
                                 */
                                *insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_X, 0, 2);
                                *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_A, BPF_REG_A);
                                *insn++ = BPF_EXIT_INSN();
                        }

                        *insn = BPF_RAW_INSN(fp->code, BPF_REG_A, BPF_REG_X, 0, fp->k);
                        break;

                /* Jump transformation cannot use BPF block macros
                 * everywhere as offset calculation and target updates
                 * require a bit more work than the rest, i.e. jump
                 * opcodes map as-is, but offsets need adjustment.
                 */

#define BPF_EMIT_JMP                                                        \
        do {                                                                \
                const s32 off_min = S16_MIN, off_max = S16_MAX;                \
                s32 off;                                                \
                                                                        \
                if (target >= len || target < 0)                        \
                        goto err;                                        \
                off = addrs ? addrs[target] - addrs[i] - 1 : 0;                \
                /* Adjust pc relative offset for 2nd or 3rd insn. */        \
                off -= insn - tmp_insns;                                \
                /* Reject anything not fitting into insn->off. */        \
                if (off < off_min || off > off_max)                        \
                        goto err;                                        \
                insn->off = off;                                        \
        } while (0)

                case BPF_JMP | BPF_JA:
                        target = i + fp->k + 1;
                        insn->code = fp->code;
                        BPF_EMIT_JMP;
                        break;

                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JEQ | BPF_X:
                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP | BPF_JSET | BPF_X:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_X:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_X:
                        if (BPF_SRC(fp->code) == BPF_K && (int) fp->k < 0) {
                                /* BPF immediates are signed, zero extend
                                 * immediate into tmp register and use it
                                 * in compare insn.
                                 */
                                *insn++ = BPF_MOV32_IMM(BPF_REG_TMP, fp->k);

                                insn->dst_reg = BPF_REG_A;
                                insn->src_reg = BPF_REG_TMP;
                                bpf_src = BPF_X;
                        } else {
                                insn->dst_reg = BPF_REG_A;
                                insn->imm = fp->k;
                                bpf_src = BPF_SRC(fp->code);
                                insn->src_reg = bpf_src == BPF_X ? BPF_REG_X : 0;
                        }

                        /* Common case where 'jump_false' is next insn. */
                        if (fp->jf == 0) {
                                insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
                                target = i + fp->jt + 1;
                                BPF_EMIT_JMP;
                                break;
                        }

                        /* Convert some jumps when 'jump_true' is next insn. */
                        if (fp->jt == 0) {
                                switch (BPF_OP(fp->code)) {
                                case BPF_JEQ:
                                        insn->code = BPF_JMP | BPF_JNE | bpf_src;
                                        break;
                                case BPF_JGT:
                                        insn->code = BPF_JMP | BPF_JLE | bpf_src;
                                        break;
                                case BPF_JGE:
                                        insn->code = BPF_JMP | BPF_JLT | bpf_src;
                                        break;
                                default:
                                        goto jmp_rest;
                                }

                                target = i + fp->jf + 1;
                                BPF_EMIT_JMP;
                                break;
                        }
jmp_rest:
                        /* Other jumps are mapped into two insns: Jxx and JA. */
                        target = i + fp->jt + 1;
                        insn->code = BPF_JMP | BPF_OP(fp->code) | bpf_src;
                        BPF_EMIT_JMP;
                        insn++;

                        insn->code = BPF_JMP | BPF_JA;
                        target = i + fp->jf + 1;
                        BPF_EMIT_JMP;
                        break;

                /* ldxb 4 * ([14] & 0xf) is remapped into 6 insns. */
                case BPF_LDX | BPF_MSH | BPF_B: {
                        struct sock_filter tmp = {
                                .code        = BPF_LD | BPF_ABS | BPF_B,
                                .k        = fp->k,
                        };

                        *seen_ld_abs = true;

                        /* X = A */
                        *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
                        /* A = BPF_R0 = *(u8 *) (skb->data + K) */
                        convert_bpf_ld_abs(&tmp, &insn);
                        insn++;
                        /* A &= 0xf */
                        *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_A, 0xf);
                        /* A <<= 2 */
                        *insn++ = BPF_ALU32_IMM(BPF_LSH, BPF_REG_A, 2);
                        /* tmp = X */
                        *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_X);
                        /* X = A */
                        *insn++ = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
                        /* A = tmp */
                        *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_TMP);
                        break;
                }
                /* RET_K is remapped into 2 insns. RET_A case doesn't need an
                 * extra mov as BPF_REG_0 is already mapped into BPF_REG_A.
                 */
                case BPF_RET | BPF_A:
                case BPF_RET | BPF_K:
                        if (BPF_RVAL(fp->code) == BPF_K)
                                *insn++ = BPF_MOV32_RAW(BPF_K, BPF_REG_0,
                                                        0, fp->k);
                        *insn = BPF_EXIT_INSN();
                        break;

                /* Store to stack. */
                case BPF_ST:
                case BPF_STX:
                        stack_off = fp->k * 4  + 4;
                        *insn = BPF_STX_MEM(BPF_W, BPF_REG_FP, BPF_CLASS(fp->code) ==
                                            BPF_ST ? BPF_REG_A : BPF_REG_X,
                                            -stack_off);
                        /* check_load_and_stores() verifies that classic BPF can
                         * load from stack only after write, so tracking
                         * stack_depth for ST|STX insns is enough
                         */
                        if (new_prog && new_prog->aux->stack_depth < stack_off)
                                new_prog->aux->stack_depth = stack_off;
                        break;

                /* Load from stack. */
                case BPF_LD | BPF_MEM:
                case BPF_LDX | BPF_MEM:
                        stack_off = fp->k * 4  + 4;
                        *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD  ?
                                            BPF_REG_A : BPF_REG_X, BPF_REG_FP,
                                            -stack_off);
                        break;

                /* A = K or X = K */
                case BPF_LD | BPF_IMM:
                case BPF_LDX | BPF_IMM:
                        *insn = BPF_MOV32_IMM(BPF_CLASS(fp->code) == BPF_LD ?
                                              BPF_REG_A : BPF_REG_X, fp->k);
                        break;

                /* X = A */
                case BPF_MISC | BPF_TAX:
                        *insn = BPF_MOV64_REG(BPF_REG_X, BPF_REG_A);
                        break;

                /* A = X */
                case BPF_MISC | BPF_TXA:
                        *insn = BPF_MOV64_REG(BPF_REG_A, BPF_REG_X);
                        break;

                /* A = skb->len or X = skb->len */
                case BPF_LD | BPF_W | BPF_LEN:
                case BPF_LDX | BPF_W | BPF_LEN:
                        *insn = BPF_LDX_MEM(BPF_W, BPF_CLASS(fp->code) == BPF_LD ?
                                            BPF_REG_A : BPF_REG_X, BPF_REG_CTX,
                                            offsetof(struct sk_buff, len));
                        break;

                /* Access seccomp_data fields. */
                case BPF_LDX | BPF_ABS | BPF_W:
                        /* A = *(u32 *) (ctx + K) */
                        *insn = BPF_LDX_MEM(BPF_W, BPF_REG_A, BPF_REG_CTX, fp->k);
                        break;

                /* Unknown instruction. */
                default:
                        goto err;
                }

                insn++;
                if (new_prog)
                        memcpy(new_insn, tmp_insns,
                               sizeof(*insn) * (insn - tmp_insns));
                new_insn += insn - tmp_insns;
        }

        if (!new_prog) {
                /* Only calculating new length. */
                *new_len = new_insn - first_insn;
                if (*seen_ld_abs)
                        *new_len += 4; /* Prologue bits. */
                return 0;
        }

        pass++;
        if (new_flen != new_insn - first_insn) {
                new_flen = new_insn - first_insn;
                if (pass > 2)
                        goto err;
                goto do_pass;
        }

        kfree(addrs);
        BUG_ON(*new_len != new_flen);
        return 0;
err:
        kfree(addrs);
        return -EINVAL;
}

/* Security:
 *
 * As we dont want to clear mem[] array for each packet going through
 * __bpf_prog_run(), we check that filter loaded by user never try to read
 * a cell if not previously written, and we check all branches to be sure
 * a malicious user doesn't try to abuse us.
 */
static int check_load_and_stores(const struct sock_filter *filter, int flen)
{
        u16 *masks, memvalid = 0; /* One bit per cell, 16 cells */
        int pc, ret = 0;

        BUILD_BUG_ON(BPF_MEMWORDS > 16);

        masks = kmalloc_array(flen, sizeof(*masks), GFP_KERNEL);
        if (!masks)
                return -ENOMEM;

        memset(masks, 0xff, flen * sizeof(*masks));

        for (pc = 0; pc < flen; pc++) {
                memvalid &= masks[pc];

                switch (filter[pc].code) {
                case BPF_ST:
                case BPF_STX:
                        memvalid |= (1 << filter[pc].k);
                        break;
                case BPF_LD | BPF_MEM:
                case BPF_LDX | BPF_MEM:
                        if (!(memvalid & (1 << filter[pc].k))) {
                                ret = -EINVAL;
                                goto error;
                        }
                        break;
                case BPF_JMP | BPF_JA:
                        /* A jump must set masks on target */
                        masks[pc + 1 + filter[pc].k] &= memvalid;
                        memvalid = ~0;
                        break;
                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JEQ | BPF_X:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_X:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_X:
                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP | BPF_JSET | BPF_X:
                        /* A jump must set masks on targets */
                        masks[pc + 1 + filter[pc].jt] &= memvalid;
                        masks[pc + 1 + filter[pc].jf] &= memvalid;
                        memvalid = ~0;
                        break;
                }
        }
error:
        kfree(masks);
        return ret;
}

static bool chk_code_allowed(u16 code_to_probe)
{
        static const bool codes[] = {
                /* 32 bit ALU operations */
                [BPF_ALU | BPF_ADD | BPF_K] = true,
                [BPF_ALU | BPF_ADD | BPF_X] = true,
                [BPF_ALU | BPF_SUB | BPF_K] = true,
                [BPF_ALU | BPF_SUB | BPF_X] = true,
                [BPF_ALU | BPF_MUL | BPF_K] = true,
                [BPF_ALU | BPF_MUL | BPF_X] = true,
                [BPF_ALU | BPF_DIV | BPF_K] = true,
                [BPF_ALU | BPF_DIV | BPF_X] = true,
                [BPF_ALU | BPF_MOD | BPF_K] = true,
                [BPF_ALU | BPF_MOD | BPF_X] = true,
                [BPF_ALU | BPF_AND | BPF_K] = true,
                [BPF_ALU | BPF_AND | BPF_X] = true,
                [BPF_ALU | BPF_OR | BPF_K] = true,
                [BPF_ALU | BPF_OR | BPF_X] = true,
                [BPF_ALU | BPF_XOR | BPF_K] = true,
                [BPF_ALU | BPF_XOR | BPF_X] = true,
                [BPF_ALU | BPF_LSH | BPF_K] = true,
                [BPF_ALU | BPF_LSH | BPF_X] = true,
                [BPF_ALU | BPF_RSH | BPF_K] = true,
                [BPF_ALU | BPF_RSH | BPF_X] = true,
                [BPF_ALU | BPF_NEG] = true,
                /* Load instructions */
                [BPF_LD | BPF_W | BPF_ABS] = true,
                [BPF_LD | BPF_H | BPF_ABS] = true,
                [BPF_LD | BPF_B | BPF_ABS] = true,
                [BPF_LD | BPF_W | BPF_LEN] = true,
                [BPF_LD | BPF_W | BPF_IND] = true,
                [BPF_LD | BPF_H | BPF_IND] = true,
                [BPF_LD | BPF_B | BPF_IND] = true,
                [BPF_LD | BPF_IMM] = true,
                [BPF_LD | BPF_MEM] = true,
                [BPF_LDX | BPF_W | BPF_LEN] = true,
                [BPF_LDX | BPF_B | BPF_MSH] = true,
                [BPF_LDX | BPF_IMM] = true,
                [BPF_LDX | BPF_MEM] = true,
                /* Store instructions */
                [BPF_ST] = true,
                [BPF_STX] = true,
                /* Misc instructions */
                [BPF_MISC | BPF_TAX] = true,
                [BPF_MISC | BPF_TXA] = true,
                /* Return instructions */
                [BPF_RET | BPF_K] = true,
                [BPF_RET | BPF_A] = true,
                /* Jump instructions */
                [BPF_JMP | BPF_JA] = true,
                [BPF_JMP | BPF_JEQ | BPF_K] = true,
                [BPF_JMP | BPF_JEQ | BPF_X] = true,
                [BPF_JMP | BPF_JGE | BPF_K] = true,
                [BPF_JMP | BPF_JGE | BPF_X] = true,
                [BPF_JMP | BPF_JGT | BPF_K] = true,
                [BPF_JMP | BPF_JGT | BPF_X] = true,
                [BPF_JMP | BPF_JSET | BPF_K] = true,
                [BPF_JMP | BPF_JSET | BPF_X] = true,
        };

        if (code_to_probe >= ARRAY_SIZE(codes))
                return false;

        return codes[code_to_probe];
}

static bool bpf_check_basics_ok(const struct sock_filter *filter,
                                unsigned int flen)
{
        if (filter == NULL)
                return false;
        if (flen == 0 || flen > BPF_MAXINSNS)
                return false;

        return true;
}

/**
 *        bpf_check_classic - verify socket filter code
 *        @filter: filter to verify
 *        @flen: length of filter
 *
 * Check the user's filter code. If we let some ugly
 * filter code slip through kaboom! The filter must contain
 * no references or jumps that are out of range, no illegal
 * instructions, and must end with a RET instruction.
 *
 * All jumps are forward as they are not signed.
 *
 * Returns 0 if the rule set is legal or -EINVAL if not.
 */
static int bpf_check_classic(const struct sock_filter *filter,
                             unsigned int flen)
{
        bool anc_found;
        int pc;

        /* Check the filter code now */
        for (pc = 0; pc < flen; pc++) {
                const struct sock_filter *ftest = &filter[pc];

                /* May we actually operate on this code? */
                if (!chk_code_allowed(ftest->code))
                        return -EINVAL;

                /* Some instructions need special checks */
                switch (ftest->code) {
                case BPF_ALU | BPF_DIV | BPF_K:
                case BPF_ALU | BPF_MOD | BPF_K:
                        /* Check for division by zero */
                        if (ftest->k == 0)
                                return -EINVAL;
                        break;
                case BPF_ALU | BPF_LSH | BPF_K:
                case BPF_ALU | BPF_RSH | BPF_K:
                        if (ftest->k >= 32)
                                return -EINVAL;
                        break;
                case BPF_LD | BPF_MEM:
                case BPF_LDX | BPF_MEM:
                case BPF_ST:
                case BPF_STX:
                        /* Check for invalid memory addresses */
                        if (ftest->k >= BPF_MEMWORDS)
                                return -EINVAL;
                        break;
                case BPF_JMP | BPF_JA:
                        /* Note, the large ftest->k might cause loops.
                         * Compare this with conditional jumps below,
                         * where offsets are limited. --ANK (981016)
                         */
                        if (ftest->k >= (unsigned int)(flen - pc - 1))
                                return -EINVAL;
                        break;
                case BPF_JMP | BPF_JEQ | BPF_K:
                case BPF_JMP | BPF_JEQ | BPF_X:
                case BPF_JMP | BPF_JGE | BPF_K:
                case BPF_JMP | BPF_JGE | BPF_X:
                case BPF_JMP | BPF_JGT | BPF_K:
                case BPF_JMP | BPF_JGT | BPF_X:
                case BPF_JMP | BPF_JSET | BPF_K:
                case BPF_JMP | BPF_JSET | BPF_X:
                        /* Both conditionals must be safe */
                        if (pc + ftest->jt + 1 >= flen ||
                            pc + ftest->jf + 1 >= flen)
                                return -EINVAL;
                        break;
                case BPF_LD | BPF_W | BPF_ABS:
                case BPF_LD | BPF_H | BPF_ABS:
                case BPF_LD | BPF_B | BPF_ABS:
                        anc_found = false;
                        if (bpf_anc_helper(ftest) & BPF_ANC)
                                anc_found = true;
                        /* Ancillary operation unknown or unsupported */
                        if (anc_found == false && ftest->k >= SKF_AD_OFF)
                                return -EINVAL;
                }
        }

        /* Last instruction must be a RET code */
        switch (filter[flen - 1].code) {
        case BPF_RET | BPF_K:
        case BPF_RET | BPF_A:
                return check_load_and_stores(filter, flen);
        }

        return -EINVAL;
}

static int bpf_prog_store_orig_filter(struct bpf_prog *fp,
                                      const struct sock_fprog *fprog)
{
        unsigned int fsize = bpf_classic_proglen(fprog);
        struct sock_fprog_kern *fkprog;

        fp->orig_prog = kmalloc(sizeof(*fkprog), GFP_KERNEL);
        if (!fp->orig_prog)
                return -ENOMEM;

        fkprog = fp->orig_prog;
        fkprog->len = fprog->len;

        fkprog->filter = kmemdup(fp->insns, fsize,
                                 GFP_KERNEL | __GFP_NOWARN);
        if (!fkprog->filter) {
                kfree(fp->orig_prog);
                return -ENOMEM;
        }

        return 0;
}

static void bpf_release_orig_filter(struct bpf_prog *fp)
{
        struct sock_fprog_kern *fprog = fp->orig_prog;

        if (fprog) {
                kfree(fprog->filter);
                kfree(fprog);
        }
}

static void __bpf_prog_release(struct bpf_prog *prog)
{
        if (prog->type == BPF_PROG_TYPE_SOCKET_FILTER) {
                bpf_prog_put(prog);
        } else {
                bpf_release_orig_filter(prog);
                bpf_prog_free(prog);
        }
}

static void __sk_filter_release(struct sk_filter *fp)
{
        __bpf_prog_release(fp->prog);
        kfree(fp);
}

/**
 *         sk_filter_release_rcu - Release a socket filter by rcu_head
 *        @rcu: rcu_head that contains the sk_filter to free
 */
static void sk_filter_release_rcu(struct rcu_head *rcu)
{
        struct sk_filter *fp = container_of(rcu, struct sk_filter, rcu);

        __sk_filter_release(fp);
}

/**
 *        sk_filter_release - release a socket filter
 *        @fp: filter to remove
 *
 *        Remove a filter from a socket and release its resources.
 */
static void sk_filter_release(struct sk_filter *fp)
{
        if (refcount_dec_and_test(&fp->refcnt))
                call_rcu(&fp->rcu, sk_filter_release_rcu);
}

void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
{
        u32 filter_size = bpf_prog_size(fp->prog->len);

        atomic_sub(filter_size, &sk->sk_omem_alloc);
        sk_filter_release(fp);
}

/* try to charge the socket memory if there is space available
 * return true on success
 */
static bool __sk_filter_charge(struct sock *sk, struct sk_filter *fp)
{
        int optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
        u32 filter_size = bpf_prog_size(fp->prog->len);

        /* same check as in sock_kmalloc() */
        if (filter_size <= optmem_max &&
            atomic_read(&sk->sk_omem_alloc) + filter_size < optmem_max) {
                atomic_add(filter_size, &sk->sk_omem_alloc);
                return true;
        }
        return false;
}

bool sk_filter_charge(struct sock *sk, struct sk_filter *fp)
{
        if (!refcount_inc_not_zero(&fp->refcnt))
                return false;

        if (!__sk_filter_charge(sk, fp)) {
                sk_filter_release(fp);
                return false;
        }
        return true;
}

static struct bpf_prog *bpf_migrate_filter(struct bpf_prog *fp)
{
        struct sock_filter *old_prog;
        struct bpf_prog *old_fp;
        int err, new_len, old_len = fp->len;
        bool seen_ld_abs = false;

        /* We are free to overwrite insns et al right here as it won't be used at
         * this point in time anymore internally after the migration to the eBPF
         * instruction representation.
         */
        BUILD_BUG_ON(sizeof(struct sock_filter) !=
                     sizeof(struct bpf_insn));

        /* Conversion cannot happen on overlapping memory areas,
         * so we need to keep the user BPF around until the 2nd
         * pass. At this time, the user BPF is stored in fp->insns.
         */
        old_prog = kmemdup_array(fp->insns, old_len, sizeof(struct sock_filter),
                                 GFP_KERNEL | __GFP_NOWARN);
        if (!old_prog) {
                err = -ENOMEM;
                goto out_err;
        }

        /* 1st pass: calculate the new program length. */
        err = bpf_convert_filter(old_prog, old_len, NULL, &new_len,
                                 &seen_ld_abs);
        if (err)
                goto out_err_free;

        /* Expand fp for appending the new filter representation. */
        old_fp = fp;
        fp = bpf_prog_realloc(old_fp, bpf_prog_size(new_len), 0);
        if (!fp) {
                /* The old_fp is still around in case we couldn't
                 * allocate new memory, so uncharge on that one.
                 */
                fp = old_fp;
                err = -ENOMEM;
                goto out_err_free;
        }

        fp->len = new_len;

        /* 2nd pass: remap sock_filter insns into bpf_insn insns. */
        err = bpf_convert_filter(old_prog, old_len, fp, &new_len,
                                 &seen_ld_abs);
        if (err)
                /* 2nd bpf_convert_filter() can fail only if it fails
                 * to allocate memory, remapping must succeed. Note,
                 * that at this time old_fp has already been released
                 * by krealloc().
                 */
                goto out_err_free;

        fp = bpf_prog_select_runtime(fp, &err);
        if (err)
                goto out_err_free;

        kfree(old_prog);
        return fp;

out_err_free:
        kfree(old_prog);
out_err:
        __bpf_prog_release(fp);
        return ERR_PTR(err);
}

static struct bpf_prog *bpf_prepare_filter(struct bpf_prog *fp,
                                           bpf_aux_classic_check_t trans)
{
        int err;

        fp->bpf_func = NULL;
        fp->jited = 0;

        err = bpf_check_classic(fp->insns, fp->len);
        if (err) {
                __bpf_prog_release(fp);
                return ERR_PTR(err);
        }

        /* There might be additional checks and transformations
         * needed on classic filters, f.e. in case of seccomp.
         */
        if (trans) {
                err = trans(fp->insns, fp->len);
                if (err) {
                        __bpf_prog_release(fp);
                        return ERR_PTR(err);
                }
        }

        /* Probe if we can JIT compile the filter and if so, do
         * the compilation of the filter.
         */
        bpf_jit_compile(fp);

        /* JIT compiler couldn't process this filter, so do the eBPF translation
         * for the optimized interpreter.
         */
        if (!fp->jited)
                fp = bpf_migrate_filter(fp);

        return fp;
}

/**
 *        bpf_prog_create - create an unattached filter
 *        @pfp: the unattached filter that is created
 *        @fprog: the filter program
 *
 * Create a filter independent of any socket. We first run some
 * sanity checks on it to make sure it does not explode on us later.
 * If an error occurs or there is insufficient memory for the filter
 * a negative errno code is returned. On success the return is zero.
 */
int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog)
{
        unsigned int fsize = bpf_classic_proglen(fprog);
        struct bpf_prog *fp;

        /* Make sure new filter is there and in the right amounts. */
        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                return -EINVAL;

        fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
        if (!fp)
                return -ENOMEM;

        memcpy(fp->insns, fprog->filter, fsize);

        fp->len = fprog->len;
        /* Since unattached filters are not copied back to user
         * space through sk_get_filter(), we do not need to hold
         * a copy here, and can spare us the work.
         */
        fp->orig_prog = NULL;

        /* bpf_prepare_filter() already takes care of freeing
         * memory in case something goes wrong.
         */
        fp = bpf_prepare_filter(fp, NULL);
        if (IS_ERR(fp))
                return PTR_ERR(fp);

        *pfp = fp;
        return 0;
}
EXPORT_SYMBOL_GPL(bpf_prog_create);

/**
 *        bpf_prog_create_from_user - create an unattached filter from user buffer
 *        @pfp: the unattached filter that is created
 *        @fprog: the filter program
 *        @trans: post-classic verifier transformation handler
 *        @save_orig: save classic BPF program
 *
 * This function effectively does the same as bpf_prog_create(), only
 * that it builds up its insns buffer from user space provided buffer.
 * It also allows for passing a bpf_aux_classic_check_t handler.
 */
int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
                              bpf_aux_classic_check_t trans, bool save_orig)
{
        unsigned int fsize = bpf_classic_proglen(fprog);
        struct bpf_prog *fp;
        int err;

        /* Make sure new filter is there and in the right amounts. */
        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                return -EINVAL;

        fp = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
        if (!fp)
                return -ENOMEM;

        if (copy_from_user(fp->insns, fprog->filter, fsize)) {
                __bpf_prog_free(fp);
                return -EFAULT;
        }

        fp->len = fprog->len;
        fp->orig_prog = NULL;

        if (save_orig) {
                err = bpf_prog_store_orig_filter(fp, fprog);
                if (err) {
                        __bpf_prog_free(fp);
                        return -ENOMEM;
                }
        }

        /* bpf_prepare_filter() already takes care of freeing
         * memory in case something goes wrong.
         */
        fp = bpf_prepare_filter(fp, trans);
        if (IS_ERR(fp))
                return PTR_ERR(fp);

        *pfp = fp;
        return 0;
}
EXPORT_SYMBOL_GPL(bpf_prog_create_from_user);

void bpf_prog_destroy(struct bpf_prog *fp)
{
        __bpf_prog_release(fp);
}
EXPORT_SYMBOL_GPL(bpf_prog_destroy);

static int __sk_attach_prog(struct bpf_prog *prog, struct sock *sk)
{
        struct sk_filter *fp, *old_fp;

        fp = kmalloc(sizeof(*fp), GFP_KERNEL);
        if (!fp)
                return -ENOMEM;

        fp->prog = prog;

        if (!__sk_filter_charge(sk, fp)) {
                kfree(fp);
                return -ENOMEM;
        }
        refcount_set(&fp->refcnt, 1);

        old_fp = rcu_dereference_protected(sk->sk_filter,
                                           lockdep_sock_is_held(sk));
        rcu_assign_pointer(sk->sk_filter, fp);

        if (old_fp)
                sk_filter_uncharge(sk, old_fp);

        return 0;
}

static
struct bpf_prog *__get_filter(struct sock_fprog *fprog, struct sock *sk)
{
        unsigned int fsize = bpf_classic_proglen(fprog);
        struct bpf_prog *prog;
        int err;

        if (sock_flag(sk, SOCK_FILTER_LOCKED))
                return ERR_PTR(-EPERM);

        /* Make sure new filter is there and in the right amounts. */
        if (!bpf_check_basics_ok(fprog->filter, fprog->len))
                return ERR_PTR(-EINVAL);

        prog = bpf_prog_alloc(bpf_prog_size(fprog->len), 0);
        if (!prog)
                return ERR_PTR(-ENOMEM);

        if (copy_from_user(prog->insns, fprog->filter, fsize)) {
                __bpf_prog_free(prog);
                return ERR_PTR(-EFAULT);
        }

        prog->len = fprog->len;

        err = bpf_prog_store_orig_filter(prog, fprog);
        if (err) {
                __bpf_prog_free(prog);
                return ERR_PTR(-ENOMEM);
        }

        /* bpf_prepare_filter() already takes care of freeing
         * memory in case something goes wrong.
         */
        return bpf_prepare_filter(prog, NULL);
}

/**
 *        sk_attach_filter - attach a socket filter
 *        @fprog: the filter program
 *        @sk: the socket to use
 *
 * Attach the user's filter code. We first run some sanity checks on
 * it to make sure it does not explode on us later. If an error
 * occurs or there is insufficient memory for the filter a negative
 * errno code is returned. On success the return is zero.
 */
int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
{
        struct bpf_prog *prog = __get_filter(fprog, sk);
        int err;

        if (IS_ERR(prog))
                return PTR_ERR(prog);

        err = __sk_attach_prog(prog, sk);
        if (err < 0) {
                __bpf_prog_release(prog);
                return err;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(sk_attach_filter);

int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk)
{
        struct bpf_prog *prog = __get_filter(fprog, sk);
        int err, optmem_max;

        if (IS_ERR(prog))
                return PTR_ERR(prog);

        optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
        if (bpf_prog_size(prog->len) > optmem_max)
                err = -ENOMEM;
        else
                err = reuseport_attach_prog(sk, prog);

        if (err)
                __bpf_prog_release(prog);

        return err;
}

static struct bpf_prog *__get_bpf(u32 ufd, struct sock *sk)
{
        if (sock_flag(sk, SOCK_FILTER_LOCKED))
                return ERR_PTR(-EPERM);

        return bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
}

int sk_attach_bpf(u32 ufd, struct sock *sk)
{
        struct bpf_prog *prog = __get_bpf(ufd, sk);
        int err;

        if (IS_ERR(prog))
                return PTR_ERR(prog);

        err = __sk_attach_prog(prog, sk);
        if (err < 0) {
                bpf_prog_put(prog);
                return err;
        }

        return 0;
}

int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk)
{
        struct bpf_prog *prog;
        int err, optmem_max;

        if (sock_flag(sk, SOCK_FILTER_LOCKED))
                return -EPERM;

        prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SOCKET_FILTER);
        if (PTR_ERR(prog) == -EINVAL)
                prog = bpf_prog_get_type(ufd, BPF_PROG_TYPE_SK_REUSEPORT);
        if (IS_ERR(prog))
                return PTR_ERR(prog);

        if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT) {
                /* Like other non BPF_PROG_TYPE_SOCKET_FILTER
                 * bpf prog (e.g. sockmap).  It depends on the
                 * limitation imposed by bpf_prog_load().
                 * Hence, sysctl_optmem_max is not checked.
                 */
                if ((sk->sk_type != SOCK_STREAM &&
                     sk->sk_type != SOCK_DGRAM) ||
                    (sk->sk_protocol != IPPROTO_UDP &&
                     sk->sk_protocol != IPPROTO_TCP) ||
                    (sk->sk_family != AF_INET &&
                     sk->sk_family != AF_INET6)) {
                        err = -ENOTSUPP;
                        goto err_prog_put;
                }
        } else {
                /* BPF_PROG_TYPE_SOCKET_FILTER */
                optmem_max = READ_ONCE(sock_net(sk)->core.sysctl_optmem_max);
                if (bpf_prog_size(prog->len) > optmem_max) {
                        err = -ENOMEM;
                        goto err_prog_put;
                }
        }

        err = reuseport_attach_prog(sk, prog);
err_prog_put:
        if (err)
                bpf_prog_put(prog);

        return err;
}

void sk_reuseport_prog_free(struct bpf_prog *prog)
{
        if (!prog)
                return;

        if (prog->type == BPF_PROG_TYPE_SK_REUSEPORT)
                bpf_prog_put(prog);
        else
                bpf_prog_destroy(prog);
}

static inline int __bpf_try_make_writable(struct sk_buff *skb,
                                          unsigned int write_len)
{
#ifdef CONFIG_DEBUG_NET
        /* Avoid a splat in pskb_may_pull_reason() */
        if (write_len > INT_MAX)
                return -EINVAL;
#endif
        return skb_ensure_writable(skb, write_len);
}

static inline int bpf_try_make_writable(struct sk_buff *skb,
                                        unsigned int write_len)
{
        int err = __bpf_try_make_writable(skb, write_len);

        bpf_compute_data_pointers(skb);
        return err;
}

static int bpf_try_make_head_writable(struct sk_buff *skb)
{
        return bpf_try_make_writable(skb, skb_headlen(skb));
}

static inline void bpf_push_mac_rcsum(struct sk_buff *skb)
{
        if (skb_at_tc_ingress(skb))
                skb_postpush_rcsum(skb, skb_mac_header(skb), skb->mac_len);
}

static inline void bpf_pull_mac_rcsum(struct sk_buff *skb)
{
        if (skb_at_tc_ingress(skb))
                skb_postpull_rcsum(skb, skb_mac_header(skb), skb->mac_len);
}

BPF_CALL_5(bpf_skb_store_bytes, struct sk_buff *, skb, u32, offset,
           const void *, from, u32, len, u64, flags)
{
        void *ptr;

        if (unlikely(flags & ~(BPF_F_RECOMPUTE_CSUM | BPF_F_INVALIDATE_HASH)))
                return -EINVAL;
        if (unlikely(offset > INT_MAX))
                return -EFAULT;
        if (unlikely(bpf_try_make_writable(skb, offset + len)))
                return -EFAULT;

        ptr = skb->data + offset;
        if (flags & BPF_F_RECOMPUTE_CSUM)
                __skb_postpull_rcsum(skb, ptr, len, offset);

        memcpy(ptr, from, len);

        if (flags & BPF_F_RECOMPUTE_CSUM)
                __skb_postpush_rcsum(skb, ptr, len, offset);
        if (flags & BPF_F_INVALIDATE_HASH)
                skb_clear_hash(skb);

        return 0;
}

static const struct bpf_func_proto bpf_skb_store_bytes_proto = {
        .func                = bpf_skb_store_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE,
        .arg5_type        = ARG_ANYTHING,
};

int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from,
                          u32 len, u64 flags)
{
        return ____bpf_skb_store_bytes(skb, offset, from, len, flags);
}

BPF_CALL_4(bpf_skb_load_bytes, const struct sk_buff *, skb, u32, offset,
           void *, to, u32, len)
{
        void *ptr;

        if (unlikely(offset > INT_MAX))
                goto err_clear;

        ptr = skb_header_pointer(skb, offset, len, to);
        if (unlikely(!ptr))
                goto err_clear;
        if (ptr != to)
                memcpy(to, ptr, len);

        return 0;
err_clear:
        memset(to, 0, len);
        return -EFAULT;
}

static const struct bpf_func_proto bpf_skb_load_bytes_proto = {
        .func                = bpf_skb_load_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
};

int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len)
{
        return ____bpf_skb_load_bytes(skb, offset, to, len);
}

BPF_CALL_4(bpf_flow_dissector_load_bytes,
           const struct bpf_flow_dissector *, ctx, u32, offset,
           void *, to, u32, len)
{
        void *ptr;

        if (unlikely(offset > 0xffff))
                goto err_clear;

        if (unlikely(!ctx->skb))
                goto err_clear;

        ptr = skb_header_pointer(ctx->skb, offset, len, to);
        if (unlikely(!ptr))
                goto err_clear;
        if (ptr != to)
                memcpy(to, ptr, len);

        return 0;
err_clear:
        memset(to, 0, len);
        return -EFAULT;
}

static const struct bpf_func_proto bpf_flow_dissector_load_bytes_proto = {
        .func                = bpf_flow_dissector_load_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_skb_load_bytes_relative, const struct sk_buff *, skb,
           u32, offset, void *, to, u32, len, u32, start_header)
{
        u8 *end = skb_tail_pointer(skb);
        u8 *start, *ptr;

        if (unlikely(offset > 0xffff))
                goto err_clear;

        switch (start_header) {
        case BPF_HDR_START_MAC:
                if (unlikely(!skb_mac_header_was_set(skb)))
                        goto err_clear;
                start = skb_mac_header(skb);
                break;
        case BPF_HDR_START_NET:
                start = skb_network_header(skb);
                break;
        default:
                goto err_clear;
        }

        ptr = start + offset;

        if (likely(ptr + len <= end)) {
                memcpy(to, ptr, len);
                return 0;
        }

err_clear:
        memset(to, 0, len);
        return -EFAULT;
}

static const struct bpf_func_proto bpf_skb_load_bytes_relative_proto = {
        .func                = bpf_skb_load_bytes_relative,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_skb_pull_data, struct sk_buff *, skb, u32, len)
{
        /* Idea is the following: should the needed direct read/write
         * test fail during runtime, we can pull in more data and redo
         * again, since implicitly, we invalidate previous checks here.
         *
         * Or, since we know how much we need to make read/writeable,
         * this can be done once at the program beginning for direct
         * access case. By this we overcome limitations of only current
         * headroom being accessible.
         */
        return bpf_try_make_writable(skb, len ? : skb_headlen(skb));
}

static const struct bpf_func_proto bpf_skb_pull_data_proto = {
        .func                = bpf_skb_pull_data,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_1(bpf_sk_fullsock, struct sock *, sk)
{
        return sk_fullsock(sk) ? (unsigned long)sk : (unsigned long)NULL;
}

static const struct bpf_func_proto bpf_sk_fullsock_proto = {
        .func                = bpf_sk_fullsock,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_SOCK_COMMON,
};

static inline int sk_skb_try_make_writable(struct sk_buff *skb,
                                           unsigned int write_len)
{
        return __bpf_try_make_writable(skb, write_len);
}

BPF_CALL_2(sk_skb_pull_data, struct sk_buff *, skb, u32, len)
{
        /* Idea is the following: should the needed direct read/write
         * test fail during runtime, we can pull in more data and redo
         * again, since implicitly, we invalidate previous checks here.
         *
         * Or, since we know how much we need to make read/writeable,
         * this can be done once at the program beginning for direct
         * access case. By this we overcome limitations of only current
         * headroom being accessible.
         */
        return sk_skb_try_make_writable(skb, len ? : skb_headlen(skb));
}

static const struct bpf_func_proto sk_skb_pull_data_proto = {
        .func                = sk_skb_pull_data,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_l3_csum_replace, struct sk_buff *, skb, u32, offset,
           u64, from, u64, to, u64, flags)
{
        __sum16 *ptr;

        if (unlikely(flags & ~(BPF_F_HDR_FIELD_MASK)))
                return -EINVAL;
        if (unlikely(offset > 0xffff || offset & 1))
                return -EFAULT;
        if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
                return -EFAULT;

        ptr = (__sum16 *)(skb->data + offset);
        switch (flags & BPF_F_HDR_FIELD_MASK) {
        case 0:
                if (unlikely(from != 0))
                        return -EINVAL;

                csum_replace_by_diff(ptr, to);
                break;
        case 2:
                csum_replace2(ptr, from, to);
                break;
        case 4:
                csum_replace4(ptr, from, to);
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static const struct bpf_func_proto bpf_l3_csum_replace_proto = {
        .func                = bpf_l3_csum_replace,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_l4_csum_replace, struct sk_buff *, skb, u32, offset,
           u64, from, u64, to, u64, flags)
{
        bool is_pseudo = flags & BPF_F_PSEUDO_HDR;
        bool is_mmzero = flags & BPF_F_MARK_MANGLED_0;
        bool do_mforce = flags & BPF_F_MARK_ENFORCE;
        bool is_ipv6   = flags & BPF_F_IPV6;
        __sum16 *ptr;

        if (unlikely(flags & ~(BPF_F_MARK_MANGLED_0 | BPF_F_MARK_ENFORCE |
                               BPF_F_PSEUDO_HDR | BPF_F_HDR_FIELD_MASK | BPF_F_IPV6)))
                return -EINVAL;
        if (unlikely(offset > 0xffff || offset & 1))
                return -EFAULT;
        if (unlikely(bpf_try_make_writable(skb, offset + sizeof(*ptr))))
                return -EFAULT;

        ptr = (__sum16 *)(skb->data + offset);
        if (is_mmzero && !do_mforce && !*ptr)
                return 0;

        switch (flags & BPF_F_HDR_FIELD_MASK) {
        case 0:
                if (unlikely(from != 0))
                        return -EINVAL;

                inet_proto_csum_replace_by_diff(ptr, skb, to, is_pseudo, is_ipv6);
                break;
        case 2:
                inet_proto_csum_replace2(ptr, skb, from, to, is_pseudo);
                break;
        case 4:
                inet_proto_csum_replace4(ptr, skb, from, to, is_pseudo);
                break;
        default:
                return -EINVAL;
        }

        if (is_mmzero && !*ptr)
                *ptr = CSUM_MANGLED_0;
        return 0;
}

static const struct bpf_func_proto bpf_l4_csum_replace_proto = {
        .func                = bpf_l4_csum_replace,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_csum_diff, __be32 *, from, u32, from_size,
           __be32 *, to, u32, to_size, __wsum, seed)
{
        /* This is quite flexible, some examples:
         *
         * from_size == 0, to_size > 0,  seed := csum --> pushing data
         * from_size > 0,  to_size == 0, seed := csum --> pulling data
         * from_size > 0,  to_size > 0,  seed := 0    --> diffing data
         *
         * Even for diffing, from_size and to_size don't need to be equal.
         */

        __wsum ret = seed;

        if (from_size && to_size)
                ret = csum_sub(csum_partial(to, to_size, ret),
                               csum_partial(from, from_size, 0));
        else if (to_size)
                ret = csum_partial(to, to_size, ret);

        else if (from_size)
                ret = ~csum_partial(from, from_size, ~ret);

        return csum_from32to16((__force unsigned int)ret);
}

static const struct bpf_func_proto bpf_csum_diff_proto = {
        .func                = bpf_csum_diff,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
        .arg2_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg3_type        = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_csum_update, struct sk_buff *, skb, __wsum, csum)
{
        /* The interface is to be used in combination with bpf_csum_diff()
         * for direct packet writes. csum rotation for alignment as well
         * as emulating csum_sub() can be done from the eBPF program.
         */
        if (skb->ip_summed == CHECKSUM_COMPLETE)
                return (skb->csum = csum_add(skb->csum, csum));

        return -ENOTSUPP;
}

static const struct bpf_func_proto bpf_csum_update_proto = {
        .func                = bpf_csum_update,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_csum_level, struct sk_buff *, skb, u64, level)
{
        /* The interface is to be used in combination with bpf_skb_adjust_room()
         * for encap/decap of packet headers when BPF_F_ADJ_ROOM_NO_CSUM_RESET
         * is passed as flags, for example.
         */
        switch (level) {
        case BPF_CSUM_LEVEL_INC:
                __skb_incr_checksum_unnecessary(skb);
                break;
        case BPF_CSUM_LEVEL_DEC:
                __skb_decr_checksum_unnecessary(skb);
                break;
        case BPF_CSUM_LEVEL_RESET:
                __skb_reset_checksum_unnecessary(skb);
                break;
        case BPF_CSUM_LEVEL_QUERY:
                return skb->ip_summed == CHECKSUM_UNNECESSARY ?
                       skb->csum_level : -EACCES;
        default:
                return -EINVAL;
        }

        return 0;
}

static const struct bpf_func_proto bpf_csum_level_proto = {
        .func                = bpf_csum_level,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

static inline int __bpf_rx_skb(struct net_device *dev, struct sk_buff *skb)
{
        return dev_forward_skb_nomtu(dev, skb);
}

static inline int __bpf_rx_skb_no_mac(struct net_device *dev,
                                      struct sk_buff *skb)
{
        int ret = ____dev_forward_skb(dev, skb, false);

        if (likely(!ret)) {
                skb->dev = dev;
                ret = netif_rx(skb);
        }

        return ret;
}

static inline int __bpf_tx_skb(struct net_device *dev, struct sk_buff *skb)
{
        int ret;

        if (dev_xmit_recursion()) {
                net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
                kfree_skb(skb);
                return -ENETDOWN;
        }

        skb->dev = dev;
        skb_set_redirected_noclear(skb, skb_at_tc_ingress(skb));
        skb_clear_tstamp(skb);

        dev_xmit_recursion_inc();
        ret = dev_queue_xmit(skb);
        dev_xmit_recursion_dec();

        return ret;
}

static int __bpf_redirect_no_mac(struct sk_buff *skb, struct net_device *dev,
                                 u32 flags)
{
        unsigned int mlen = skb_network_offset(skb);

        if (unlikely(skb->len <= mlen)) {
                kfree_skb(skb);
                return -ERANGE;
        }

        if (mlen) {
                __skb_pull(skb, mlen);

                /* At ingress, the mac header has already been pulled once.
                 * At egress, skb_pospull_rcsum has to be done in case that
                 * the skb is originated from ingress (i.e. a forwarded skb)
                 * to ensure that rcsum starts at net header.
                 */
                if (!skb_at_tc_ingress(skb))
                        skb_postpull_rcsum(skb, skb_mac_header(skb), mlen);
        }
        skb_pop_mac_header(skb);
        skb_reset_mac_len(skb);
        return flags & BPF_F_INGRESS ?
               __bpf_rx_skb_no_mac(dev, skb) : __bpf_tx_skb(dev, skb);
}

static int __bpf_redirect_common(struct sk_buff *skb, struct net_device *dev,
                                 u32 flags)
{
        /* Verify that a link layer header is carried */
        if (unlikely(skb->mac_header >= skb->network_header || skb->len == 0)) {
                kfree_skb(skb);
                return -ERANGE;
        }

        bpf_push_mac_rcsum(skb);
        return flags & BPF_F_INGRESS ?
               __bpf_rx_skb(dev, skb) : __bpf_tx_skb(dev, skb);
}

static int __bpf_redirect(struct sk_buff *skb, struct net_device *dev,
                          u32 flags)
{
        if (dev_is_mac_header_xmit(dev))
                return __bpf_redirect_common(skb, dev, flags);
        else
                return __bpf_redirect_no_mac(skb, dev, flags);
}

#if IS_ENABLED(CONFIG_IPV6)
static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
                            struct net_device *dev, struct bpf_nh_params *nh)
{
        u32 hh_len = LL_RESERVED_SPACE(dev);
        const struct in6_addr *nexthop;
        struct dst_entry *dst = NULL;
        struct neighbour *neigh;

        if (dev_xmit_recursion()) {
                net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
                goto out_drop;
        }

        skb->dev = dev;
        skb_clear_tstamp(skb);

        if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
                skb = skb_expand_head(skb, hh_len);
                if (!skb)
                        return -ENOMEM;
        }

        rcu_read_lock();
        if (!nh) {
                dst = skb_dst(skb);
                nexthop = rt6_nexthop(dst_rt6_info(dst),
                                      &ipv6_hdr(skb)->daddr);
        } else {
                nexthop = &nh->ipv6_nh;
        }
        neigh = ip_neigh_gw6(dev, nexthop);
        if (likely(!IS_ERR(neigh))) {
                int ret;

                sock_confirm_neigh(skb, neigh);
                local_bh_disable();
                dev_xmit_recursion_inc();
                ret = neigh_output(neigh, skb, false);
                dev_xmit_recursion_dec();
                local_bh_enable();
                rcu_read_unlock();
                return ret;
        }
        rcu_read_unlock();
        if (dst)
                IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
out_drop:
        kfree_skb(skb);
        return -ENETDOWN;
}

static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
                                   struct bpf_nh_params *nh)
{
        const struct ipv6hdr *ip6h = ipv6_hdr(skb);
        struct net *net = dev_net(dev);
        int err, ret = NET_XMIT_DROP;

        if (!nh) {
                struct dst_entry *dst;
                struct flowi6 fl6 = {
                        .flowi6_flags = FLOWI_FLAG_ANYSRC,
                        .flowi6_mark  = skb->mark,
                        .flowlabel    = ip6_flowinfo(ip6h),
                        .flowi6_oif   = dev->ifindex,
                        .flowi6_proto = ip6h->nexthdr,
                        .daddr              = ip6h->daddr,
                        .saddr              = ip6h->saddr,
                };

                dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
                if (IS_ERR(dst))
                        goto out_drop;

                skb_dst_set(skb, dst);
        } else if (nh->nh_family != AF_INET6) {
                goto out_drop;
        }

        err = bpf_out_neigh_v6(net, skb, dev, nh);
        if (unlikely(net_xmit_eval(err)))
                DEV_STATS_INC(dev, tx_errors);
        else
                ret = NET_XMIT_SUCCESS;
        goto out_xmit;
out_drop:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
out_xmit:
        return ret;
}
#else
static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
                                   struct bpf_nh_params *nh)
{
        kfree_skb(skb);
        return NET_XMIT_DROP;
}
#endif /* CONFIG_IPV6 */

#if IS_ENABLED(CONFIG_INET)
static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
                            struct net_device *dev, struct bpf_nh_params *nh)
{
        u32 hh_len = LL_RESERVED_SPACE(dev);
        struct neighbour *neigh;
        bool is_v6gw = false;

        if (dev_xmit_recursion()) {
                net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
                goto out_drop;
        }

        skb->dev = dev;
        skb_clear_tstamp(skb);

        if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
                skb = skb_expand_head(skb, hh_len);
                if (!skb)
                        return -ENOMEM;
        }

        rcu_read_lock();
        if (!nh) {
                struct rtable *rt = skb_rtable(skb);

                neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
        } else if (nh->nh_family == AF_INET6) {
                neigh = ip_neigh_gw6(dev, &nh->ipv6_nh);
                is_v6gw = true;
        } else if (nh->nh_family == AF_INET) {
                neigh = ip_neigh_gw4(dev, nh->ipv4_nh);
        } else {
                rcu_read_unlock();
                goto out_drop;
        }

        if (likely(!IS_ERR(neigh))) {
                int ret;

                sock_confirm_neigh(skb, neigh);
                local_bh_disable();
                dev_xmit_recursion_inc();
                ret = neigh_output(neigh, skb, is_v6gw);
                dev_xmit_recursion_dec();
                local_bh_enable();
                rcu_read_unlock();
                return ret;
        }
        rcu_read_unlock();
out_drop:
        kfree_skb(skb);
        return -ENETDOWN;
}

static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
                                   struct bpf_nh_params *nh)
{
        const struct iphdr *ip4h = ip_hdr(skb);
        struct net *net = dev_net(dev);
        int err, ret = NET_XMIT_DROP;

        if (!nh) {
                struct flowi4 fl4 = {
                        .flowi4_flags = FLOWI_FLAG_ANYSRC,
                        .flowi4_mark  = skb->mark,
                        .flowi4_tos   = inet_dscp_to_dsfield(ip4h_dscp(ip4h)),
                        .flowi4_oif   = dev->ifindex,
                        .flowi4_proto = ip4h->protocol,
                        .daddr              = ip4h->daddr,
                        .saddr              = ip4h->saddr,
                };
                struct rtable *rt;

                rt = ip_route_output_flow(net, &fl4, NULL);
                if (IS_ERR(rt))
                        goto out_drop;
                if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
                        ip_rt_put(rt);
                        goto out_drop;
                }

                skb_dst_set(skb, &rt->dst);
        }

        err = bpf_out_neigh_v4(net, skb, dev, nh);
        if (unlikely(net_xmit_eval(err)))
                DEV_STATS_INC(dev, tx_errors);
        else
                ret = NET_XMIT_SUCCESS;
        goto out_xmit;
out_drop:
        DEV_STATS_INC(dev, tx_errors);
        kfree_skb(skb);
out_xmit:
        return ret;
}
#else
static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
                                   struct bpf_nh_params *nh)
{
        kfree_skb(skb);
        return NET_XMIT_DROP;
}
#endif /* CONFIG_INET */

static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev,
                                struct bpf_nh_params *nh)
{
        struct ethhdr *ethh = eth_hdr(skb);

        if (unlikely(skb->mac_header >= skb->network_header))
                goto out;
        bpf_push_mac_rcsum(skb);
        if (is_multicast_ether_addr(ethh->h_dest))
                goto out;

        skb_pull(skb, sizeof(*ethh));
        skb_unset_mac_header(skb);
        skb_reset_network_header(skb);

        if (skb->protocol == htons(ETH_P_IP))
                return __bpf_redirect_neigh_v4(skb, dev, nh);
        else if (skb->protocol == htons(ETH_P_IPV6))
                return __bpf_redirect_neigh_v6(skb, dev, nh);
out:
        kfree_skb(skb);
        return -ENOTSUPP;
}

/* Internal, non-exposed redirect flags. */
enum {
        BPF_F_NEIGH        = (1ULL << 16),
        BPF_F_PEER        = (1ULL << 17),
        BPF_F_NEXTHOP        = (1ULL << 18),
#define BPF_F_REDIRECT_INTERNAL        (BPF_F_NEIGH | BPF_F_PEER | BPF_F_NEXTHOP)
};

BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
{
        struct net_device *dev;
        struct sk_buff *clone;
        int ret;

        BUILD_BUG_ON(BPF_F_REDIRECT_INTERNAL & BPF_F_REDIRECT_FLAGS);

        if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
                return -EINVAL;

        dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex);
        if (unlikely(!dev))
                return -EINVAL;

        clone = skb_clone(skb, GFP_ATOMIC);
        if (unlikely(!clone))
                return -ENOMEM;

        /* For direct write, we need to keep the invariant that the skbs
         * we're dealing with need to be uncloned. Should uncloning fail
         * here, we need to free the just generated clone to unclone once
         * again.
         */
        ret = bpf_try_make_head_writable(skb);
        if (unlikely(ret)) {
                kfree_skb(clone);
                return -ENOMEM;
        }

        return __bpf_redirect(clone, dev, flags);
}

static const struct bpf_func_proto bpf_clone_redirect_proto = {
        .func           = bpf_clone_redirect,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_ANYTHING,
};

static struct net_device *skb_get_peer_dev(struct net_device *dev)
{
        const struct net_device_ops *ops = dev->netdev_ops;

        if (likely(ops->ndo_get_peer_dev))
                return INDIRECT_CALL_1(ops->ndo_get_peer_dev,
                                       netkit_peer_dev, dev);
        return NULL;
}

int skb_do_redirect(struct sk_buff *skb)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        struct net *net = dev_net(skb->dev);
        struct net_device *dev;
        u32 flags = ri->flags;

        dev = dev_get_by_index_rcu(net, ri->tgt_index);
        ri->tgt_index = 0;
        ri->flags = 0;
        if (unlikely(!dev))
                goto out_drop;
        if (flags & BPF_F_PEER) {
                if (unlikely(!skb_at_tc_ingress(skb)))
                        goto out_drop;
                dev = skb_get_peer_dev(dev);
                if (unlikely(!dev ||
                             !(dev->flags & IFF_UP) ||
                             net_eq(net, dev_net(dev))))
                        goto out_drop;
                skb->dev = dev;
                dev_sw_netstats_rx_add(dev, skb->len);
                skb_scrub_packet(skb, false);
                return -EAGAIN;
        }
        return flags & BPF_F_NEIGH ?
               __bpf_redirect_neigh(skb, dev, flags & BPF_F_NEXTHOP ?
                                    &ri->nh : NULL) :
               __bpf_redirect(skb, dev, flags);
out_drop:
        kfree_skb(skb);
        return -EINVAL;
}

BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();

        if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
                return TC_ACT_SHOT;

        ri->flags = flags;
        ri->tgt_index = ifindex;

        return TC_ACT_REDIRECT;
}

static const struct bpf_func_proto bpf_redirect_proto = {
        .func           = bpf_redirect,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_ANYTHING,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();

        if (unlikely(flags))
                return TC_ACT_SHOT;

        ri->flags = BPF_F_PEER;
        ri->tgt_index = ifindex;

        return TC_ACT_REDIRECT;
}

static const struct bpf_func_proto bpf_redirect_peer_proto = {
        .func           = bpf_redirect_peer,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_ANYTHING,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params,
           int, plen, u64, flags)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();

        if (unlikely((plen && plen < sizeof(*params)) || flags))
                return TC_ACT_SHOT;

        ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0);
        ri->tgt_index = ifindex;

        BUILD_BUG_ON(sizeof(struct bpf_redir_neigh) != sizeof(struct bpf_nh_params));
        if (plen)
                memcpy(&ri->nh, params, sizeof(ri->nh));

        return TC_ACT_REDIRECT;
}

static const struct bpf_func_proto bpf_redirect_neigh_proto = {
        .func                = bpf_redirect_neigh,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_ANYTHING,
        .arg2_type      = ARG_PTR_TO_MEM | PTR_MAYBE_NULL | MEM_RDONLY,
        .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes)
{
        msg->apply_bytes = bytes;
        return 0;
}

static const struct bpf_func_proto bpf_msg_apply_bytes_proto = {
        .func           = bpf_msg_apply_bytes,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_2(bpf_msg_cork_bytes, struct sk_msg *, msg, u32, bytes)
{
        msg->cork_bytes = bytes;
        return 0;
}

static void sk_msg_reset_curr(struct sk_msg *msg)
{
        if (!msg->sg.size) {
                msg->sg.curr = msg->sg.start;
                msg->sg.copybreak = 0;
        } else {
                u32 i = msg->sg.end;

                sk_msg_iter_var_prev(i);
                msg->sg.curr = i;
                msg->sg.copybreak = msg->sg.data[i].length;
        }
}

static const struct bpf_func_proto bpf_msg_cork_bytes_proto = {
        .func           = bpf_msg_cork_bytes,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_4(bpf_msg_pull_data, struct sk_msg *, msg, u32, start,
           u32, end, u64, flags)
{
        u32 len = 0, offset = 0, copy = 0, poffset = 0, bytes = end - start;
        u32 first_sge, last_sge, i, shift, bytes_sg_total;
        struct scatterlist *sge;
        u8 *raw, *to, *from;
        struct page *page;

        if (unlikely(flags || end <= start))
                return -EINVAL;

        /* First find the starting scatterlist element */
        i = msg->sg.start;
        do {
                offset += len;
                len = sk_msg_elem(msg, i)->length;
                if (start < offset + len)
                        break;
                sk_msg_iter_var_next(i);
        } while (i != msg->sg.end);

        if (unlikely(start >= offset + len))
                return -EINVAL;

        first_sge = i;
        /* The start may point into the sg element so we need to also
         * account for the headroom.
         */
        bytes_sg_total = start - offset + bytes;
        if (!test_bit(i, msg->sg.copy) && bytes_sg_total <= len)
                goto out;

        /* At this point we need to linearize multiple scatterlist
         * elements or a single shared page. Either way we need to
         * copy into a linear buffer exclusively owned by BPF. Then
         * place the buffer in the scatterlist and fixup the original
         * entries by removing the entries now in the linear buffer
         * and shifting the remaining entries. For now we do not try
         * to copy partial entries to avoid complexity of running out
         * of sg_entry slots. The downside is reading a single byte
         * will copy the entire sg entry.
         */
        do {
                copy += sk_msg_elem(msg, i)->length;
                sk_msg_iter_var_next(i);
                if (bytes_sg_total <= copy)
                        break;
        } while (i != msg->sg.end);
        last_sge = i;

        if (unlikely(bytes_sg_total > copy))
                return -EINVAL;

        page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
                           get_order(copy));
        if (unlikely(!page))
                return -ENOMEM;

        raw = page_address(page);
        i = first_sge;
        do {
                sge = sk_msg_elem(msg, i);
                from = sg_virt(sge);
                len = sge->length;
                to = raw + poffset;

                memcpy(to, from, len);
                poffset += len;
                sge->length = 0;
                put_page(sg_page(sge));

                sk_msg_iter_var_next(i);
        } while (i != last_sge);

        sg_set_page(&msg->sg.data[first_sge], page, copy, 0);

        /* To repair sg ring we need to shift entries. If we only
         * had a single entry though we can just replace it and
         * be done. Otherwise walk the ring and shift the entries.
         */
        WARN_ON_ONCE(last_sge == first_sge);
        shift = last_sge > first_sge ?
                last_sge - first_sge - 1 :
                NR_MSG_FRAG_IDS - first_sge + last_sge - 1;
        if (!shift)
                goto out;

        i = first_sge;
        sk_msg_iter_var_next(i);
        do {
                u32 move_from;

                if (i + shift >= NR_MSG_FRAG_IDS)
                        move_from = i + shift - NR_MSG_FRAG_IDS;
                else
                        move_from = i + shift;
                if (move_from == msg->sg.end)
                        break;

                msg->sg.data[i] = msg->sg.data[move_from];
                msg->sg.data[move_from].length = 0;
                msg->sg.data[move_from].page_link = 0;
                msg->sg.data[move_from].offset = 0;
                sk_msg_iter_var_next(i);
        } while (1);

        msg->sg.end = msg->sg.end - shift > msg->sg.end ?
                      msg->sg.end - shift + NR_MSG_FRAG_IDS :
                      msg->sg.end - shift;
out:
        sk_msg_reset_curr(msg);
        msg->data = sg_virt(&msg->sg.data[first_sge]) + start - offset;
        msg->data_end = msg->data + bytes;
        return 0;
}

static const struct bpf_func_proto bpf_msg_pull_data_proto = {
        .func                = bpf_msg_pull_data,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_msg_push_data, struct sk_msg *, msg, u32, start,
           u32, len, u64, flags)
{
        struct scatterlist sge, nsge, nnsge, rsge = {0}, *psge;
        u32 new, i = 0, l = 0, space, copy = 0, offset = 0;
        u8 *raw, *to, *from;
        struct page *page;

        if (unlikely(flags))
                return -EINVAL;

        if (unlikely(len == 0))
                return 0;

        /* First find the starting scatterlist element */
        i = msg->sg.start;
        do {
                offset += l;
                l = sk_msg_elem(msg, i)->length;

                if (start < offset + l)
                        break;
                sk_msg_iter_var_next(i);
        } while (i != msg->sg.end);

        if (start > offset + l)
                return -EINVAL;

        space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);

        /* If no space available will fallback to copy, we need at
         * least one scatterlist elem available to push data into
         * when start aligns to the beginning of an element or two
         * when it falls inside an element. We handle the start equals
         * offset case because its the common case for inserting a
         * header.
         */
        if (!space || (space == 1 && start != offset))
                copy = msg->sg.data[i].length;

        page = alloc_pages(__GFP_NOWARN | GFP_ATOMIC | __GFP_COMP,
                           get_order(copy + len));
        if (unlikely(!page))
                return -ENOMEM;

        if (copy) {
                int front, back;

                raw = page_address(page);

                if (i == msg->sg.end)
                        sk_msg_iter_var_prev(i);
                psge = sk_msg_elem(msg, i);
                front = start - offset;
                back = psge->length - front;
                from = sg_virt(psge);

                if (front)
                        memcpy(raw, from, front);

                if (back) {
                        from += front;
                        to = raw + front + len;

                        memcpy(to, from, back);
                }

                put_page(sg_page(psge));
                new = i;
                goto place_new;
        }

        if (start - offset) {
                if (i == msg->sg.end)
                        sk_msg_iter_var_prev(i);
                psge = sk_msg_elem(msg, i);
                rsge = sk_msg_elem_cpy(msg, i);

                psge->length = start - offset;
                rsge.length -= psge->length;
                rsge.offset += start;

                sk_msg_iter_var_next(i);
                sg_unmark_end(psge);
                sg_unmark_end(&rsge);
        }

        /* Slot(s) to place newly allocated data */
        sk_msg_iter_next(msg, end);
        new = i;
        sk_msg_iter_var_next(i);

        if (i == msg->sg.end) {
                if (!rsge.length)
                        goto place_new;
                sk_msg_iter_next(msg, end);
                goto place_new;
        }

        /* Shift one or two slots as needed */
        sge = sk_msg_elem_cpy(msg, new);
        sg_unmark_end(&sge);

        nsge = sk_msg_elem_cpy(msg, i);
        if (rsge.length) {
                sk_msg_iter_var_next(i);
                nnsge = sk_msg_elem_cpy(msg, i);
                sk_msg_iter_next(msg, end);
        }

        while (i != msg->sg.end) {
                msg->sg.data[i] = sge;
                sge = nsge;
                sk_msg_iter_var_next(i);
                if (rsge.length) {
                        nsge = nnsge;
                        nnsge = sk_msg_elem_cpy(msg, i);
                } else {
                        nsge = sk_msg_elem_cpy(msg, i);
                }
        }

place_new:
        /* Place newly allocated data buffer */
        sk_mem_charge(msg->sk, len);
        msg->sg.size += len;
        __clear_bit(new, msg->sg.copy);
        sg_set_page(&msg->sg.data[new], page, len + copy, 0);
        if (rsge.length) {
                get_page(sg_page(&rsge));
                sk_msg_iter_var_next(new);
                msg->sg.data[new] = rsge;
        }

        sk_msg_reset_curr(msg);
        sk_msg_compute_data_pointers(msg);
        return 0;
}

static const struct bpf_func_proto bpf_msg_push_data_proto = {
        .func                = bpf_msg_push_data,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
};

static void sk_msg_shift_left(struct sk_msg *msg, int i)
{
        struct scatterlist *sge = sk_msg_elem(msg, i);
        int prev;

        put_page(sg_page(sge));
        do {
                prev = i;
                sk_msg_iter_var_next(i);
                msg->sg.data[prev] = msg->sg.data[i];
        } while (i != msg->sg.end);

        sk_msg_iter_prev(msg, end);
}

static void sk_msg_shift_right(struct sk_msg *msg, int i)
{
        struct scatterlist tmp, sge;

        sk_msg_iter_next(msg, end);
        sge = sk_msg_elem_cpy(msg, i);
        sk_msg_iter_var_next(i);
        tmp = sk_msg_elem_cpy(msg, i);

        while (i != msg->sg.end) {
                msg->sg.data[i] = sge;
                sk_msg_iter_var_next(i);
                sge = tmp;
                tmp = sk_msg_elem_cpy(msg, i);
        }
}

BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,
           u32, len, u64, flags)
{
        u32 i = 0, l = 0, space, offset = 0;
        u64 last = start + len;
        int pop;

        if (unlikely(flags))
                return -EINVAL;

        if (unlikely(len == 0))
                return 0;

        /* First find the starting scatterlist element */
        i = msg->sg.start;
        do {
                offset += l;
                l = sk_msg_elem(msg, i)->length;

                if (start < offset + l)
                        break;
                sk_msg_iter_var_next(i);
        } while (i != msg->sg.end);

        /* Bounds checks: start and pop must be inside message */
        if (start >= offset + l || last > msg->sg.size)
                return -EINVAL;

        space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);

        pop = len;
        /* --------------| offset
         * -| start      |-------- len -------|
         *
         *  |----- a ----|-------- pop -------|----- b ----|
         *  |______________________________________________| length
         *
         *
         * a:   region at front of scatter element to save
         * b:   region at back of scatter element to save when length > A + pop
         * pop: region to pop from element, same as input 'pop' here will be
         *      decremented below per iteration.
         *
         * Two top-level cases to handle when start != offset, first B is non
         * zero and second B is zero corresponding to when a pop includes more
         * than one element.
         *
         * Then if B is non-zero AND there is no space allocate space and
         * compact A, B regions into page. If there is space shift ring to
         * the right free'ing the next element in ring to place B, leaving
         * A untouched except to reduce length.
         */
        if (start != offset) {
                struct scatterlist *nsge, *sge = sk_msg_elem(msg, i);
                int a = start - offset;
                int b = sge->length - pop - a;

                sk_msg_iter_var_next(i);

                if (b > 0) {
                        if (space) {
                                sge->length = a;
                                sk_msg_shift_right(msg, i);
                                nsge = sk_msg_elem(msg, i);
                                get_page(sg_page(sge));
                                sg_set_page(nsge,
                                            sg_page(sge),
                                            b, sge->offset + pop + a);
                        } else {
                                struct page *page, *orig;
                                u8 *to, *from;

                                page = alloc_pages(__GFP_NOWARN |
                                                   __GFP_COMP   | GFP_ATOMIC,
                                                   get_order(a + b));
                                if (unlikely(!page))
                                        return -ENOMEM;

                                orig = sg_page(sge);
                                from = sg_virt(sge);
                                to = page_address(page);
                                memcpy(to, from, a);
                                memcpy(to + a, from + a + pop, b);
                                sg_set_page(sge, page, a + b, 0);
                                put_page(orig);
                        }
                        pop = 0;
                } else {
                        pop -= (sge->length - a);
                        sge->length = a;
                }
        }

        /* From above the current layout _must_ be as follows,
         *
         * -| offset
         * -| start
         *
         *  |---- pop ---|---------------- b ------------|
         *  |____________________________________________| length
         *
         * Offset and start of the current msg elem are equal because in the
         * previous case we handled offset != start and either consumed the
         * entire element and advanced to the next element OR pop == 0.
         *
         * Two cases to handle here are first pop is less than the length
         * leaving some remainder b above. Simply adjust the element's layout
         * in this case. Or pop >= length of the element so that b = 0. In this
         * case advance to next element decrementing pop.
         */
        while (pop) {
                struct scatterlist *sge = sk_msg_elem(msg, i);

                if (pop < sge->length) {
                        sge->length -= pop;
                        sge->offset += pop;
                        pop = 0;
                } else {
                        pop -= sge->length;
                        sk_msg_shift_left(msg, i);
                }
        }

        sk_mem_uncharge(msg->sk, len - pop);
        msg->sg.size -= (len - pop);
        sk_msg_reset_curr(msg);
        sk_msg_compute_data_pointers(msg);
        return 0;
}

static const struct bpf_func_proto bpf_msg_pop_data_proto = {
        .func                = bpf_msg_pop_data,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
};

#ifdef CONFIG_CGROUP_NET_CLASSID
BPF_CALL_0(bpf_get_cgroup_classid_curr)
{
        return __task_get_classid(current);
}

const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
        .func                = bpf_get_cgroup_classid_curr,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
};

BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb)
{
        struct sock *sk = skb_to_full_sk(skb);

        if (!sk || !sk_fullsock(sk))
                return 0;

        return sock_cgroup_classid(&sk->sk_cgrp_data);
}

static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = {
        .func                = bpf_skb_cgroup_classid,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};
#endif

BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
{
        return task_get_classid(skb);
}

static const struct bpf_func_proto bpf_get_cgroup_classid_proto = {
        .func           = bpf_get_cgroup_classid,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_get_route_realm, const struct sk_buff *, skb)
{
        return dst_tclassid(skb);
}

static const struct bpf_func_proto bpf_get_route_realm_proto = {
        .func           = bpf_get_route_realm,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_get_hash_recalc, struct sk_buff *, skb)
{
        /* If skb_clear_hash() was called due to mangling, we can
         * trigger SW recalculation here. Later access to hash
         * can then use the inline skb->hash via context directly
         * instead of calling this helper again.
         */
        return skb_get_hash(skb);
}

static const struct bpf_func_proto bpf_get_hash_recalc_proto = {
        .func                = bpf_get_hash_recalc,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_set_hash_invalid, struct sk_buff *, skb)
{
        /* After all direct packet write, this can be used once for
         * triggering a lazy recalc on next skb_get_hash() invocation.
         */
        skb_clear_hash(skb);
        return 0;
}

static const struct bpf_func_proto bpf_set_hash_invalid_proto = {
        .func                = bpf_set_hash_invalid,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

BPF_CALL_2(bpf_set_hash, struct sk_buff *, skb, u32, hash)
{
        /* Set user specified hash as L4(+), so that it gets returned
         * on skb_get_hash() call unless BPF prog later on triggers a
         * skb_clear_hash().
         */
        __skb_set_sw_hash(skb, hash, true);
        return 0;
}

static const struct bpf_func_proto bpf_set_hash_proto = {
        .func                = bpf_set_hash,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_3(bpf_skb_vlan_push, struct sk_buff *, skb, __be16, vlan_proto,
           u16, vlan_tci)
{
        int ret;

        if (unlikely(vlan_proto != htons(ETH_P_8021Q) &&
                     vlan_proto != htons(ETH_P_8021AD)))
                vlan_proto = htons(ETH_P_8021Q);

        bpf_push_mac_rcsum(skb);
        ret = skb_vlan_push(skb, vlan_proto, vlan_tci);
        bpf_pull_mac_rcsum(skb);
        skb_reset_mac_len(skb);

        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_vlan_push_proto = {
        .func           = bpf_skb_vlan_push,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_ANYTHING,
};

BPF_CALL_1(bpf_skb_vlan_pop, struct sk_buff *, skb)
{
        int ret;

        bpf_push_mac_rcsum(skb);
        ret = skb_vlan_pop(skb);
        bpf_pull_mac_rcsum(skb);

        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_vlan_pop_proto = {
        .func           = bpf_skb_vlan_pop,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

static void bpf_skb_change_protocol(struct sk_buff *skb, u16 proto)
{
        skb->protocol = htons(proto);
        if (skb_valid_dst(skb))
                skb_dst_drop(skb);
}

static int bpf_skb_generic_push(struct sk_buff *skb, u32 off, u32 len)
{
        /* Caller already did skb_cow() with len as headroom,
         * so no need to do it here.
         */
        skb_push(skb, len);
        memmove(skb->data, skb->data + len, off);
        memset(skb->data + off, 0, len);

        /* No skb_postpush_rcsum(skb, skb->data + off, len)
         * needed here as it does not change the skb->csum
         * result for checksum complete when summing over
         * zeroed blocks.
         */
        return 0;
}

static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
{
        void *old_data;

        /* skb_ensure_writable() is not needed here, as we're
         * already working on an uncloned skb.
         */
        if (unlikely(!pskb_may_pull(skb, off + len)))
                return -ENOMEM;

        old_data = skb->data;
        __skb_pull(skb, len);
        skb_postpull_rcsum(skb, old_data + off, len);
        memmove(skb->data, old_data, off);

        return 0;
}

static int bpf_skb_net_hdr_push(struct sk_buff *skb, u32 off, u32 len)
{
        bool trans_same = skb->transport_header == skb->network_header;
        int ret;

        /* There's no need for __skb_push()/__skb_pull() pair to
         * get to the start of the mac header as we're guaranteed
         * to always start from here under eBPF.
         */
        ret = bpf_skb_generic_push(skb, off, len);
        if (likely(!ret)) {
                skb->mac_header -= len;
                skb->network_header -= len;
                if (trans_same)
                        skb->transport_header = skb->network_header;
        }

        return ret;
}

static int bpf_skb_net_hdr_pop(struct sk_buff *skb, u32 off, u32 len)
{
        bool trans_same = skb->transport_header == skb->network_header;
        int ret;

        /* Same here, __skb_push()/__skb_pull() pair not needed. */
        ret = bpf_skb_generic_pop(skb, off, len);
        if (likely(!ret)) {
                skb->mac_header += len;
                skb->network_header += len;
                if (trans_same)
                        skb->transport_header = skb->network_header;
        }

        return ret;
}

static int bpf_skb_proto_4_to_6(struct sk_buff *skb)
{
        const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
        u32 off = skb_mac_header_len(skb);
        int ret;

        ret = skb_cow(skb, len_diff);
        if (unlikely(ret < 0))
                return ret;

        ret = bpf_skb_net_hdr_push(skb, off, len_diff);
        if (unlikely(ret < 0))
                return ret;

        if (skb_is_gso(skb)) {
                struct skb_shared_info *shinfo = skb_shinfo(skb);

                /* SKB_GSO_TCPV4 needs to be changed into SKB_GSO_TCPV6. */
                if (shinfo->gso_type & SKB_GSO_TCPV4) {
                        shinfo->gso_type &= ~SKB_GSO_TCPV4;
                        shinfo->gso_type |=  SKB_GSO_TCPV6;
                }
        }

        bpf_skb_change_protocol(skb, ETH_P_IPV6);
        skb_clear_hash(skb);

        return 0;
}

static int bpf_skb_proto_6_to_4(struct sk_buff *skb)
{
        const u32 len_diff = sizeof(struct ipv6hdr) - sizeof(struct iphdr);
        u32 off = skb_mac_header_len(skb);
        int ret;

        ret = skb_unclone(skb, GFP_ATOMIC);
        if (unlikely(ret < 0))
                return ret;

        ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
        if (unlikely(ret < 0))
                return ret;

        if (skb_is_gso(skb)) {
                struct skb_shared_info *shinfo = skb_shinfo(skb);

                /* SKB_GSO_TCPV6 needs to be changed into SKB_GSO_TCPV4. */
                if (shinfo->gso_type & SKB_GSO_TCPV6) {
                        shinfo->gso_type &= ~SKB_GSO_TCPV6;
                        shinfo->gso_type |=  SKB_GSO_TCPV4;
                }
        }

        bpf_skb_change_protocol(skb, ETH_P_IP);
        skb_clear_hash(skb);

        return 0;
}

static int bpf_skb_proto_xlat(struct sk_buff *skb, __be16 to_proto)
{
        __be16 from_proto = skb->protocol;

        if (from_proto == htons(ETH_P_IP) &&
              to_proto == htons(ETH_P_IPV6))
                return bpf_skb_proto_4_to_6(skb);

        if (from_proto == htons(ETH_P_IPV6) &&
              to_proto == htons(ETH_P_IP))
                return bpf_skb_proto_6_to_4(skb);

        return -ENOTSUPP;
}

BPF_CALL_3(bpf_skb_change_proto, struct sk_buff *, skb, __be16, proto,
           u64, flags)
{
        int ret;

        if (unlikely(flags))
                return -EINVAL;

        /* General idea is that this helper does the basic groundwork
         * needed for changing the protocol, and eBPF program fills the
         * rest through bpf_skb_store_bytes(), bpf_lX_csum_replace()
         * and other helpers, rather than passing a raw buffer here.
         *
         * The rationale is to keep this minimal and without a need to
         * deal with raw packet data. F.e. even if we would pass buffers
         * here, the program still needs to call the bpf_lX_csum_replace()
         * helpers anyway. Plus, this way we keep also separation of
         * concerns, since f.e. bpf_skb_store_bytes() should only take
         * care of stores.
         *
         * Currently, additional options and extension header space are
         * not supported, but flags register is reserved so we can adapt
         * that. For offloads, we mark packet as dodgy, so that headers
         * need to be verified first.
         */
        ret = bpf_skb_proto_xlat(skb, proto);
        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_change_proto_proto = {
        .func                = bpf_skb_change_proto,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_skb_change_type, struct sk_buff *, skb, u32, pkt_type)
{
        /* We only allow a restricted subset to be changed for now. */
        if (unlikely(!skb_pkt_type_ok(skb->pkt_type) ||
                     !skb_pkt_type_ok(pkt_type)))
                return -EINVAL;

        skb->pkt_type = pkt_type;
        return 0;
}

static const struct bpf_func_proto bpf_skb_change_type_proto = {
        .func                = bpf_skb_change_type,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

static u32 bpf_skb_net_base_len(const struct sk_buff *skb)
{
        switch (skb->protocol) {
        case htons(ETH_P_IP):
                return sizeof(struct iphdr);
        case htons(ETH_P_IPV6):
                return sizeof(struct ipv6hdr);
        default:
                return ~0U;
        }
}

#define BPF_F_ADJ_ROOM_ENCAP_L3_MASK        (BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 | \
                                         BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)

#define BPF_F_ADJ_ROOM_DECAP_L3_MASK        (BPF_F_ADJ_ROOM_DECAP_L3_IPV4 | \
                                         BPF_F_ADJ_ROOM_DECAP_L3_IPV6)

#define BPF_F_ADJ_ROOM_MASK                (BPF_F_ADJ_ROOM_FIXED_GSO | \
                                         BPF_F_ADJ_ROOM_ENCAP_L3_MASK | \
                                         BPF_F_ADJ_ROOM_ENCAP_L4_GRE | \
                                         BPF_F_ADJ_ROOM_ENCAP_L4_UDP | \
                                         BPF_F_ADJ_ROOM_ENCAP_L2_ETH | \
                                         BPF_F_ADJ_ROOM_ENCAP_L2( \
                                          BPF_ADJ_ROOM_ENCAP_L2_MASK) | \
                                         BPF_F_ADJ_ROOM_DECAP_L3_MASK)

static int bpf_skb_net_grow(struct sk_buff *skb, u32 off, u32 len_diff,
                            u64 flags)
{
        u8 inner_mac_len = flags >> BPF_ADJ_ROOM_ENCAP_L2_SHIFT;
        bool encap = flags & BPF_F_ADJ_ROOM_ENCAP_L3_MASK;
        u16 mac_len = 0, inner_net = 0, inner_trans = 0;
        unsigned int gso_type = SKB_GSO_DODGY;
        int ret;

        if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
                /* udp gso_size delineates datagrams, only allow if fixed */
                if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
                    !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
                        return -ENOTSUPP;
        }

        ret = skb_cow_head(skb, len_diff);
        if (unlikely(ret < 0))
                return ret;

        if (encap) {
                if (skb->protocol != htons(ETH_P_IP) &&
                    skb->protocol != htons(ETH_P_IPV6))
                        return -ENOTSUPP;

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 &&
                    flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
                        return -EINVAL;

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE &&
                    flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
                        return -EINVAL;

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH &&
                    inner_mac_len < ETH_HLEN)
                        return -EINVAL;

                if (skb->encapsulation)
                        return -EALREADY;

                mac_len = skb->network_header - skb->mac_header;
                inner_net = skb->network_header;
                if (inner_mac_len > len_diff)
                        return -EINVAL;
                inner_trans = skb->transport_header;
        }

        ret = bpf_skb_net_hdr_push(skb, off, len_diff);
        if (unlikely(ret < 0))
                return ret;

        if (encap) {
                skb->inner_mac_header = inner_net - inner_mac_len;
                skb->inner_network_header = inner_net;
                skb->inner_transport_header = inner_trans;

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L2_ETH)
                        skb_set_inner_protocol(skb, htons(ETH_P_TEB));
                else
                        skb_set_inner_protocol(skb, skb->protocol);

                skb->encapsulation = 1;
                skb_set_network_header(skb, mac_len);

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP)
                        gso_type |= SKB_GSO_UDP_TUNNEL;
                else if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE)
                        gso_type |= SKB_GSO_GRE;
                else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
                        gso_type |= SKB_GSO_IPXIP6;
                else if (flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
                        gso_type |= SKB_GSO_IPXIP4;

                if (flags & BPF_F_ADJ_ROOM_ENCAP_L4_GRE ||
                    flags & BPF_F_ADJ_ROOM_ENCAP_L4_UDP) {
                        int nh_len = flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 ?
                                        sizeof(struct ipv6hdr) :
                                        sizeof(struct iphdr);

                        skb_set_transport_header(skb, mac_len + nh_len);
                }

                /* Match skb->protocol to new outer l3 protocol */
                if (skb->protocol == htons(ETH_P_IP) &&
                    flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV6)
                        bpf_skb_change_protocol(skb, ETH_P_IPV6);
                else if (skb->protocol == htons(ETH_P_IPV6) &&
                         flags & BPF_F_ADJ_ROOM_ENCAP_L3_IPV4)
                        bpf_skb_change_protocol(skb, ETH_P_IP);
        }

        if (skb_is_gso(skb)) {
                struct skb_shared_info *shinfo = skb_shinfo(skb);

                /* Header must be checked, and gso_segs recomputed. */
                shinfo->gso_type |= gso_type;
                shinfo->gso_segs = 0;

                /* Due to header growth, MSS needs to be downgraded.
                 * There is a BUG_ON() when segmenting the frag_list with
                 * head_frag true, so linearize the skb after downgrading
                 * the MSS.
                 */
                if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO)) {
                        skb_decrease_gso_size(shinfo, len_diff);
                        if (shinfo->frag_list)
                                return skb_linearize(skb);
                }
        }

        return 0;
}

static int bpf_skb_net_shrink(struct sk_buff *skb, u32 off, u32 len_diff,
                              u64 flags)
{
        int ret;

        if (unlikely(flags & ~(BPF_F_ADJ_ROOM_FIXED_GSO |
                               BPF_F_ADJ_ROOM_DECAP_L3_MASK |
                               BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
                return -EINVAL;

        if (skb_is_gso(skb) && !skb_is_gso_tcp(skb)) {
                /* udp gso_size delineates datagrams, only allow if fixed */
                if (!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ||
                    !(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
                        return -ENOTSUPP;
        }

        ret = skb_unclone(skb, GFP_ATOMIC);
        if (unlikely(ret < 0))
                return ret;

        ret = bpf_skb_net_hdr_pop(skb, off, len_diff);
        if (unlikely(ret < 0))
                return ret;

        /* Match skb->protocol to new outer l3 protocol */
        if (skb->protocol == htons(ETH_P_IP) &&
            flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV6)
                bpf_skb_change_protocol(skb, ETH_P_IPV6);
        else if (skb->protocol == htons(ETH_P_IPV6) &&
                 flags & BPF_F_ADJ_ROOM_DECAP_L3_IPV4)
                bpf_skb_change_protocol(skb, ETH_P_IP);

        if (skb_is_gso(skb)) {
                struct skb_shared_info *shinfo = skb_shinfo(skb);

                /* Due to header shrink, MSS can be upgraded. */
                if (!(flags & BPF_F_ADJ_ROOM_FIXED_GSO))
                        skb_increase_gso_size(shinfo, len_diff);

                /* Header must be checked, and gso_segs recomputed. */
                shinfo->gso_type |= SKB_GSO_DODGY;
                shinfo->gso_segs = 0;
        }

        return 0;
}

#define BPF_SKB_MAX_LEN SKB_MAX_ALLOC

BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
           u32, mode, u64, flags)
{
        u32 len_diff_abs = abs(len_diff);
        bool shrink = len_diff < 0;
        int ret = 0;

        if (unlikely(flags || mode))
                return -EINVAL;
        if (unlikely(len_diff_abs > 0xfffU))
                return -EFAULT;

        if (!shrink) {
                ret = skb_cow(skb, len_diff);
                if (unlikely(ret < 0))
                        return ret;
                __skb_push(skb, len_diff_abs);
                memset(skb->data, 0, len_diff_abs);
        } else {
                if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
                        return -ENOMEM;
                __skb_pull(skb, len_diff_abs);
        }
        if (tls_sw_has_ctx_rx(skb->sk)) {
                struct strp_msg *rxm = strp_msg(skb);

                rxm->full_len += len_diff;
        }
        return ret;
}

static const struct bpf_func_proto sk_skb_adjust_room_proto = {
        .func                = sk_skb_adjust_room,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
           u32, mode, u64, flags)
{
        u32 len_cur, len_diff_abs = abs(len_diff);
        u32 len_min = bpf_skb_net_base_len(skb);
        u32 len_max = BPF_SKB_MAX_LEN;
        __be16 proto = skb->protocol;
        bool shrink = len_diff < 0;
        u32 off;
        int ret;

        if (unlikely(flags & ~(BPF_F_ADJ_ROOM_MASK |
                               BPF_F_ADJ_ROOM_NO_CSUM_RESET)))
                return -EINVAL;
        if (unlikely(len_diff_abs > 0xfffU))
                return -EFAULT;
        if (unlikely(proto != htons(ETH_P_IP) &&
                     proto != htons(ETH_P_IPV6)))
                return -ENOTSUPP;

        off = skb_mac_header_len(skb);
        switch (mode) {
        case BPF_ADJ_ROOM_NET:
                off += bpf_skb_net_base_len(skb);
                break;
        case BPF_ADJ_ROOM_MAC:
                break;
        default:
                return -ENOTSUPP;
        }

        if (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) {
                if (!shrink)
                        return -EINVAL;

                switch (flags & BPF_F_ADJ_ROOM_DECAP_L3_MASK) {
                case BPF_F_ADJ_ROOM_DECAP_L3_IPV4:
                        len_min = sizeof(struct iphdr);
                        break;
                case BPF_F_ADJ_ROOM_DECAP_L3_IPV6:
                        len_min = sizeof(struct ipv6hdr);
                        break;
                default:
                        return -EINVAL;
                }
        }

        len_cur = skb->len - skb_network_offset(skb);
        if ((shrink && (len_diff_abs >= len_cur ||
                        len_cur - len_diff_abs < len_min)) ||
            (!shrink && (skb->len + len_diff_abs > len_max &&
                         !skb_is_gso(skb))))
                return -ENOTSUPP;

        ret = shrink ? bpf_skb_net_shrink(skb, off, len_diff_abs, flags) :
                       bpf_skb_net_grow(skb, off, len_diff_abs, flags);
        if (!ret && !(flags & BPF_F_ADJ_ROOM_NO_CSUM_RESET))
                __skb_reset_checksum_unnecessary(skb);

        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_adjust_room_proto = {
        .func                = bpf_skb_adjust_room,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_ANYTHING,
};

static u32 __bpf_skb_min_len(const struct sk_buff *skb)
{
        int offset = skb_network_offset(skb);
        u32 min_len = 0;

        if (offset > 0)
                min_len = offset;
        if (skb_transport_header_was_set(skb)) {
                offset = skb_transport_offset(skb);
                if (offset > 0)
                        min_len = offset;
        }
        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                offset = skb_checksum_start_offset(skb) +
                         skb->csum_offset + sizeof(__sum16);
                if (offset > 0)
                        min_len = offset;
        }
        return min_len;
}

static int bpf_skb_grow_rcsum(struct sk_buff *skb, unsigned int new_len)
{
        unsigned int old_len = skb->len;
        int ret;

        ret = __skb_grow_rcsum(skb, new_len);
        if (!ret)
                memset(skb->data + old_len, 0, new_len - old_len);
        return ret;
}

static int bpf_skb_trim_rcsum(struct sk_buff *skb, unsigned int new_len)
{
        return __skb_trim_rcsum(skb, new_len);
}

static inline int __bpf_skb_change_tail(struct sk_buff *skb, u32 new_len,
                                        u64 flags)
{
        u32 max_len = BPF_SKB_MAX_LEN;
        u32 min_len = __bpf_skb_min_len(skb);
        int ret;

        if (unlikely(flags || new_len > max_len || new_len < min_len))
                return -EINVAL;
        if (skb->encapsulation)
                return -ENOTSUPP;

        /* The basic idea of this helper is that it's performing the
         * needed work to either grow or trim an skb, and eBPF program
         * rewrites the rest via helpers like bpf_skb_store_bytes(),
         * bpf_lX_csum_replace() and others rather than passing a raw
         * buffer here. This one is a slow path helper and intended
         * for replies with control messages.
         *
         * Like in bpf_skb_change_proto(), we want to keep this rather
         * minimal and without protocol specifics so that we are able
         * to separate concerns as in bpf_skb_store_bytes() should only
         * be the one responsible for writing buffers.
         *
         * It's really expected to be a slow path operation here for
         * control message replies, so we're implicitly linearizing,
         * uncloning and drop offloads from the skb by this.
         */
        ret = __bpf_try_make_writable(skb, skb->len);
        if (!ret) {
                if (new_len > skb->len)
                        ret = bpf_skb_grow_rcsum(skb, new_len);
                else if (new_len < skb->len)
                        ret = bpf_skb_trim_rcsum(skb, new_len);
                if (!ret && skb_is_gso(skb))
                        skb_gso_reset(skb);
        }
        return ret;
}

BPF_CALL_3(bpf_skb_change_tail, struct sk_buff *, skb, u32, new_len,
           u64, flags)
{
        int ret = __bpf_skb_change_tail(skb, new_len, flags);

        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_change_tail_proto = {
        .func                = bpf_skb_change_tail,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

BPF_CALL_3(sk_skb_change_tail, struct sk_buff *, skb, u32, new_len,
           u64, flags)
{
        return __bpf_skb_change_tail(skb, new_len, flags);
}

static const struct bpf_func_proto sk_skb_change_tail_proto = {
        .func                = sk_skb_change_tail,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

static inline int __bpf_skb_change_head(struct sk_buff *skb, u32 head_room,
                                        u64 flags)
{
        u32 max_len = BPF_SKB_MAX_LEN;
        u32 new_len = skb->len + head_room;
        int ret;

        if (unlikely(flags || (!skb_is_gso(skb) && new_len > max_len) ||
                     new_len < skb->len))
                return -EINVAL;

        ret = skb_cow(skb, head_room);
        if (likely(!ret)) {
                /* Idea for this helper is that we currently only
                 * allow to expand on mac header. This means that
                 * skb->protocol network header, etc, stay as is.
                 * Compared to bpf_skb_change_tail(), we're more
                 * flexible due to not needing to linearize or
                 * reset GSO. Intention for this helper is to be
                 * used by an L3 skb that needs to push mac header
                 * for redirection into L2 device.
                 */
                __skb_push(skb, head_room);
                memset(skb->data, 0, head_room);
                skb_reset_mac_header(skb);
                skb_reset_mac_len(skb);
        }

        return ret;
}

BPF_CALL_3(bpf_skb_change_head, struct sk_buff *, skb, u32, head_room,
           u64, flags)
{
        int ret = __bpf_skb_change_head(skb, head_room, flags);

        bpf_compute_data_pointers(skb);
        return ret;
}

static const struct bpf_func_proto bpf_skb_change_head_proto = {
        .func                = bpf_skb_change_head,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

BPF_CALL_3(sk_skb_change_head, struct sk_buff *, skb, u32, head_room,
           u64, flags)
{
        return __bpf_skb_change_head(skb, head_room, flags);
}

static const struct bpf_func_proto sk_skb_change_head_proto = {
        .func                = sk_skb_change_head,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

BPF_CALL_1(bpf_xdp_get_buff_len, struct xdp_buff*, xdp)
{
        return xdp_get_buff_len(xdp);
}

static const struct bpf_func_proto bpf_xdp_get_buff_len_proto = {
        .func                = bpf_xdp_get_buff_len,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

BTF_ID_LIST_SINGLE(bpf_xdp_get_buff_len_bpf_ids, struct, xdp_buff)

const struct bpf_func_proto bpf_xdp_get_buff_len_trace_proto = {
        .func                = bpf_xdp_get_buff_len,
        .gpl_only        = false,
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &bpf_xdp_get_buff_len_bpf_ids[0],
};

static unsigned long xdp_get_metalen(const struct xdp_buff *xdp)
{
        return xdp_data_meta_unsupported(xdp) ? 0 :
               xdp->data - xdp->data_meta;
}

BPF_CALL_2(bpf_xdp_adjust_head, struct xdp_buff *, xdp, int, offset)
{
        void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
        unsigned long metalen = xdp_get_metalen(xdp);
        void *data_start = xdp_frame_end + metalen;
        void *data = xdp->data + offset;

        if (unlikely(data < data_start ||
                     data > xdp->data_end - ETH_HLEN))
                return -EINVAL;

        if (metalen)
                memmove(xdp->data_meta + offset,
                        xdp->data_meta, metalen);
        xdp->data_meta += offset;
        xdp->data = data;

        return 0;
}

static const struct bpf_func_proto bpf_xdp_adjust_head_proto = {
        .func                = bpf_xdp_adjust_head,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

void bpf_xdp_copy_buf(struct xdp_buff *xdp, unsigned long off,
                      void *buf, unsigned long len, bool flush)
{
        unsigned long ptr_len, ptr_off = 0;
        skb_frag_t *next_frag, *end_frag;
        struct skb_shared_info *sinfo;
        void *src, *dst;
        u8 *ptr_buf;

        if (likely(xdp->data_end - xdp->data >= off + len)) {
                src = flush ? buf : xdp->data + off;
                dst = flush ? xdp->data + off : buf;
                memcpy(dst, src, len);
                return;
        }

        sinfo = xdp_get_shared_info_from_buff(xdp);
        end_frag = &sinfo->frags[sinfo->nr_frags];
        next_frag = &sinfo->frags[0];

        ptr_len = xdp->data_end - xdp->data;
        ptr_buf = xdp->data;

        while (true) {
                if (off < ptr_off + ptr_len) {
                        unsigned long copy_off = off - ptr_off;
                        unsigned long copy_len = min(len, ptr_len - copy_off);

                        src = flush ? buf : ptr_buf + copy_off;
                        dst = flush ? ptr_buf + copy_off : buf;
                        memcpy(dst, src, copy_len);

                        off += copy_len;
                        len -= copy_len;
                        buf += copy_len;
                }

                if (!len || next_frag == end_frag)
                        break;

                ptr_off += ptr_len;
                ptr_buf = skb_frag_address(next_frag);
                ptr_len = skb_frag_size(next_frag);
                next_frag++;
        }
}

void *bpf_xdp_pointer(struct xdp_buff *xdp, u32 offset, u32 len)
{
        u32 size = xdp->data_end - xdp->data;
        struct skb_shared_info *sinfo;
        void *addr = xdp->data;
        int i;

        if (unlikely(offset > 0xffff || len > 0xffff))
                return ERR_PTR(-EFAULT);

        if (unlikely(offset + len > xdp_get_buff_len(xdp)))
                return ERR_PTR(-EINVAL);

        if (likely(offset < size)) /* linear area */
                goto out;

        sinfo = xdp_get_shared_info_from_buff(xdp);
        offset -= size;
        for (i = 0; i < sinfo->nr_frags; i++) { /* paged area */
                u32 frag_size = skb_frag_size(&sinfo->frags[i]);

                if  (offset < frag_size) {
                        addr = skb_frag_address(&sinfo->frags[i]);
                        size = frag_size;
                        break;
                }
                offset -= frag_size;
        }
out:
        return offset + len <= size ? addr + offset : NULL;
}

BPF_CALL_4(bpf_xdp_load_bytes, struct xdp_buff *, xdp, u32, offset,
           void *, buf, u32, len)
{
        void *ptr;

        ptr = bpf_xdp_pointer(xdp, offset, len);
        if (IS_ERR(ptr))
                return PTR_ERR(ptr);

        if (!ptr)
                bpf_xdp_copy_buf(xdp, offset, buf, len, false);
        else
                memcpy(buf, ptr, len);

        return 0;
}

static const struct bpf_func_proto bpf_xdp_load_bytes_proto = {
        .func                = bpf_xdp_load_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
};

int __bpf_xdp_load_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
{
        return ____bpf_xdp_load_bytes(xdp, offset, buf, len);
}

BPF_CALL_4(bpf_xdp_store_bytes, struct xdp_buff *, xdp, u32, offset,
           void *, buf, u32, len)
{
        void *ptr;

        ptr = bpf_xdp_pointer(xdp, offset, len);
        if (IS_ERR(ptr))
                return PTR_ERR(ptr);

        if (!ptr)
                bpf_xdp_copy_buf(xdp, offset, buf, len, true);
        else
                memcpy(ptr, buf, len);

        return 0;
}

static const struct bpf_func_proto bpf_xdp_store_bytes_proto = {
        .func                = bpf_xdp_store_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
};

int __bpf_xdp_store_bytes(struct xdp_buff *xdp, u32 offset, void *buf, u32 len)
{
        return ____bpf_xdp_store_bytes(xdp, offset, buf, len);
}

static int bpf_xdp_frags_increase_tail(struct xdp_buff *xdp, int offset)
{
        struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
        skb_frag_t *frag = &sinfo->frags[sinfo->nr_frags - 1];
        struct xdp_rxq_info *rxq = xdp->rxq;
        unsigned int tailroom;

        if (!rxq->frag_size || rxq->frag_size > xdp->frame_sz)
                return -EOPNOTSUPP;

        tailroom = rxq->frag_size - skb_frag_size(frag) - skb_frag_off(frag);
        if (unlikely(offset > tailroom))
                return -EINVAL;

        memset(skb_frag_address(frag) + skb_frag_size(frag), 0, offset);
        skb_frag_size_add(frag, offset);
        sinfo->xdp_frags_size += offset;
        if (rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL)
                xsk_buff_get_tail(xdp)->data_end += offset;

        return 0;
}

static void bpf_xdp_shrink_data_zc(struct xdp_buff *xdp, int shrink,
                                   enum xdp_mem_type mem_type, bool release)
{
        struct xdp_buff *zc_frag = xsk_buff_get_tail(xdp);

        if (release) {
                xsk_buff_del_tail(zc_frag);
                __xdp_return(0, mem_type, false, zc_frag);
        } else {
                zc_frag->data_end -= shrink;
        }
}

static bool bpf_xdp_shrink_data(struct xdp_buff *xdp, skb_frag_t *frag,
                                int shrink)
{
        enum xdp_mem_type mem_type = xdp->rxq->mem.type;
        bool release = skb_frag_size(frag) == shrink;

        if (mem_type == MEM_TYPE_XSK_BUFF_POOL) {
                bpf_xdp_shrink_data_zc(xdp, shrink, mem_type, release);
                goto out;
        }

        if (release)
                __xdp_return(skb_frag_netmem(frag), mem_type, false, NULL);

out:
        return release;
}

static int bpf_xdp_frags_shrink_tail(struct xdp_buff *xdp, int offset)
{
        struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
        int i, n_frags_free = 0, len_free = 0;

        if (unlikely(offset > (int)xdp_get_buff_len(xdp) - ETH_HLEN))
                return -EINVAL;

        for (i = sinfo->nr_frags - 1; i >= 0 && offset > 0; i--) {
                skb_frag_t *frag = &sinfo->frags[i];
                int shrink = min_t(int, offset, skb_frag_size(frag));

                len_free += shrink;
                offset -= shrink;
                if (bpf_xdp_shrink_data(xdp, frag, shrink)) {
                        n_frags_free++;
                } else {
                        skb_frag_size_sub(frag, shrink);
                        break;
                }
        }
        sinfo->nr_frags -= n_frags_free;
        sinfo->xdp_frags_size -= len_free;

        if (unlikely(!sinfo->nr_frags)) {
                xdp_buff_clear_frags_flag(xdp);
                xdp->data_end -= offset;
        }

        return 0;
}

BPF_CALL_2(bpf_xdp_adjust_tail, struct xdp_buff *, xdp, int, offset)
{
        void *data_hard_end = xdp_data_hard_end(xdp); /* use xdp->frame_sz */
        void *data_end = xdp->data_end + offset;

        if (unlikely(xdp_buff_has_frags(xdp))) { /* non-linear xdp buff */
                if (offset < 0)
                        return bpf_xdp_frags_shrink_tail(xdp, -offset);

                return bpf_xdp_frags_increase_tail(xdp, offset);
        }

        /* Notice that xdp_data_hard_end have reserved some tailroom */
        if (unlikely(data_end > data_hard_end))
                return -EINVAL;

        if (unlikely(data_end < xdp->data + ETH_HLEN))
                return -EINVAL;

        /* Clear memory area on grow, can contain uninit kernel memory */
        if (offset > 0)
                memset(xdp->data_end, 0, offset);

        xdp->data_end = data_end;

        return 0;
}

static const struct bpf_func_proto bpf_xdp_adjust_tail_proto = {
        .func                = bpf_xdp_adjust_tail,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
{
        void *xdp_frame_end = xdp->data_hard_start + sizeof(struct xdp_frame);
        void *meta = xdp->data_meta + offset;
        unsigned long metalen = xdp->data - meta;

        if (xdp_data_meta_unsupported(xdp))
                return -ENOTSUPP;
        if (unlikely(meta < xdp_frame_end ||
                     meta > xdp->data))
                return -EINVAL;
        if (unlikely(xdp_metalen_invalid(metalen)))
                return -EACCES;

        xdp->data_meta = meta;

        return 0;
}

static const struct bpf_func_proto bpf_xdp_adjust_meta_proto = {
        .func                = bpf_xdp_adjust_meta,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

/**
 * DOC: xdp redirect
 *
 * XDP_REDIRECT works by a three-step process, implemented in the functions
 * below:
 *
 * 1. The bpf_redirect() and bpf_redirect_map() helpers will lookup the target
 *    of the redirect and store it (along with some other metadata) in a per-CPU
 *    struct bpf_redirect_info.
 *
 * 2. When the program returns the XDP_REDIRECT return code, the driver will
 *    call xdp_do_redirect() which will use the information in struct
 *    bpf_redirect_info to actually enqueue the frame into a map type-specific
 *    bulk queue structure.
 *
 * 3. Before exiting its NAPI poll loop, the driver will call
 *    xdp_do_flush(), which will flush all the different bulk queues,
 *    thus completing the redirect. Note that xdp_do_flush() must be
 *    called before napi_complete_done() in the driver, as the
 *    XDP_REDIRECT logic relies on being inside a single NAPI instance
 *    through to the xdp_do_flush() call for RCU protection of all
 *    in-kernel data structures.
 */
/*
 * Pointers to the map entries will be kept around for this whole sequence of
 * steps, protected by RCU. However, there is no top-level rcu_read_lock() in
 * the core code; instead, the RCU protection relies on everything happening
 * inside a single NAPI poll sequence, which means it's between a pair of calls
 * to local_bh_disable()/local_bh_enable().
 *
 * The map entries are marked as __rcu and the map code makes sure to
 * dereference those pointers with rcu_dereference_check() in a way that works
 * for both sections that to hold an rcu_read_lock() and sections that are
 * called from NAPI without a separate rcu_read_lock(). The code below does not
 * use RCU annotations, but relies on those in the map code.
 */
void xdp_do_flush(void)
{
        struct list_head *lh_map, *lh_dev, *lh_xsk;

        bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
        if (lh_dev)
                __dev_flush(lh_dev);
        if (lh_map)
                __cpu_map_flush(lh_map);
        if (lh_xsk)
                __xsk_map_flush(lh_xsk);
}
EXPORT_SYMBOL_GPL(xdp_do_flush);

#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
void xdp_do_check_flushed(struct napi_struct *napi)
{
        struct list_head *lh_map, *lh_dev, *lh_xsk;
        bool missed = false;

        bpf_net_ctx_get_all_used_flush_lists(&lh_map, &lh_dev, &lh_xsk);
        if (lh_dev) {
                __dev_flush(lh_dev);
                missed = true;
        }
        if (lh_map) {
                __cpu_map_flush(lh_map);
                missed = true;
        }
        if (lh_xsk) {
                __xsk_map_flush(lh_xsk);
                missed = true;
        }

        WARN_ONCE(missed, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
                  napi->poll);
}
#endif

DEFINE_STATIC_KEY_FALSE(bpf_master_redirect_enabled_key);
EXPORT_SYMBOL_GPL(bpf_master_redirect_enabled_key);

u32 xdp_master_redirect(struct xdp_buff *xdp)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        struct net_device *master, *slave;

        master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev);
        slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp);
        if (slave && slave != xdp->rxq->dev) {
                /* The target device is different from the receiving device, so
                 * redirect it to the new device.
                 * Using XDP_REDIRECT gets the correct behaviour from XDP enabled
                 * drivers to unmap the packet from their rx ring.
                 */
                ri->tgt_index = slave->ifindex;
                ri->map_id = INT_MAX;
                ri->map_type = BPF_MAP_TYPE_UNSPEC;
                return XDP_REDIRECT;
        }
        return XDP_TX;
}
EXPORT_SYMBOL_GPL(xdp_master_redirect);

static inline int __xdp_do_redirect_xsk(struct bpf_redirect_info *ri,
                                        const struct net_device *dev,
                                        struct xdp_buff *xdp,
                                        const struct bpf_prog *xdp_prog)
{
        enum bpf_map_type map_type = ri->map_type;
        void *fwd = ri->tgt_value;
        u32 map_id = ri->map_id;
        int err;

        ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
        ri->map_type = BPF_MAP_TYPE_UNSPEC;

        err = __xsk_map_redirect(fwd, xdp);
        if (unlikely(err))
                goto err;

        _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
        return 0;
err:
        _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
        return err;
}

static __always_inline int
__xdp_do_redirect_frame(struct bpf_redirect_info *ri, struct net_device *dev,
                        struct xdp_frame *xdpf,
                        const struct bpf_prog *xdp_prog)
{
        enum bpf_map_type map_type = ri->map_type;
        void *fwd = ri->tgt_value;
        u32 map_id = ri->map_id;
        u32 flags = ri->flags;
        struct bpf_map *map;
        int err;

        ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
        ri->flags = 0;
        ri->map_type = BPF_MAP_TYPE_UNSPEC;

        if (unlikely(!xdpf)) {
                err = -EOVERFLOW;
                goto err;
        }

        switch (map_type) {
        case BPF_MAP_TYPE_DEVMAP:
                fallthrough;
        case BPF_MAP_TYPE_DEVMAP_HASH:
                if (unlikely(flags & BPF_F_BROADCAST)) {
                        map = READ_ONCE(ri->map);

                        /* The map pointer is cleared when the map is being torn
                         * down by dev_map_free()
                         */
                        if (unlikely(!map)) {
                                err = -ENOENT;
                                break;
                        }

                        WRITE_ONCE(ri->map, NULL);
                        err = dev_map_enqueue_multi(xdpf, dev, map,
                                                    flags & BPF_F_EXCLUDE_INGRESS);
                } else {
                        err = dev_map_enqueue(fwd, xdpf, dev);
                }
                break;
        case BPF_MAP_TYPE_CPUMAP:
                err = cpu_map_enqueue(fwd, xdpf, dev);
                break;
        case BPF_MAP_TYPE_UNSPEC:
                if (map_id == INT_MAX) {
                        fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
                        if (unlikely(!fwd)) {
                                err = -EINVAL;
                                break;
                        }
                        err = dev_xdp_enqueue(fwd, xdpf, dev);
                        break;
                }
                fallthrough;
        default:
                err = -EBADRQC;
        }

        if (unlikely(err))
                goto err;

        _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
        return 0;
err:
        _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
        return err;
}

int xdp_do_redirect(struct net_device *dev, struct xdp_buff *xdp,
                    const struct bpf_prog *xdp_prog)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        enum bpf_map_type map_type = ri->map_type;

        if (map_type == BPF_MAP_TYPE_XSKMAP)
                return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);

        return __xdp_do_redirect_frame(ri, dev, xdp_convert_buff_to_frame(xdp),
                                       xdp_prog);
}
EXPORT_SYMBOL_GPL(xdp_do_redirect);

int xdp_do_redirect_frame(struct net_device *dev, struct xdp_buff *xdp,
                          struct xdp_frame *xdpf,
                          const struct bpf_prog *xdp_prog)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        enum bpf_map_type map_type = ri->map_type;

        if (map_type == BPF_MAP_TYPE_XSKMAP)
                return __xdp_do_redirect_xsk(ri, dev, xdp, xdp_prog);

        return __xdp_do_redirect_frame(ri, dev, xdpf, xdp_prog);
}
EXPORT_SYMBOL_GPL(xdp_do_redirect_frame);

static int xdp_do_generic_redirect_map(struct net_device *dev,
                                       struct sk_buff *skb,
                                       struct xdp_buff *xdp,
                                       const struct bpf_prog *xdp_prog,
                                       void *fwd, enum bpf_map_type map_type,
                                       u32 map_id, u32 flags)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        struct bpf_map *map;
        int err;

        switch (map_type) {
        case BPF_MAP_TYPE_DEVMAP:
                fallthrough;
        case BPF_MAP_TYPE_DEVMAP_HASH:
                if (unlikely(flags & BPF_F_BROADCAST)) {
                        map = READ_ONCE(ri->map);

                        /* The map pointer is cleared when the map is being torn
                         * down by dev_map_free()
                         */
                        if (unlikely(!map)) {
                                err = -ENOENT;
                                break;
                        }

                        WRITE_ONCE(ri->map, NULL);
                        err = dev_map_redirect_multi(dev, skb, xdp_prog, map,
                                                     flags & BPF_F_EXCLUDE_INGRESS);
                } else {
                        err = dev_map_generic_redirect(fwd, skb, xdp_prog);
                }
                if (unlikely(err))
                        goto err;
                break;
        case BPF_MAP_TYPE_XSKMAP:
                err = xsk_generic_rcv(fwd, xdp);
                if (err)
                        goto err;
                consume_skb(skb);
                break;
        case BPF_MAP_TYPE_CPUMAP:
                err = cpu_map_generic_redirect(fwd, skb);
                if (unlikely(err))
                        goto err;
                break;
        default:
                err = -EBADRQC;
                goto err;
        }

        _trace_xdp_redirect_map(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index);
        return 0;
err:
        _trace_xdp_redirect_map_err(dev, xdp_prog, fwd, map_type, map_id, ri->tgt_index, err);
        return err;
}

int xdp_do_generic_redirect(struct net_device *dev, struct sk_buff *skb,
                            struct xdp_buff *xdp,
                            const struct bpf_prog *xdp_prog)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();
        enum bpf_map_type map_type = ri->map_type;
        void *fwd = ri->tgt_value;
        u32 map_id = ri->map_id;
        u32 flags = ri->flags;
        int err;

        ri->map_id = 0; /* Valid map id idr range: [1,INT_MAX[ */
        ri->flags = 0;
        ri->map_type = BPF_MAP_TYPE_UNSPEC;

        if (map_type == BPF_MAP_TYPE_UNSPEC && map_id == INT_MAX) {
                fwd = dev_get_by_index_rcu(dev_net(dev), ri->tgt_index);
                if (unlikely(!fwd)) {
                        err = -EINVAL;
                        goto err;
                }

                err = xdp_ok_fwd_dev(fwd, skb->len);
                if (unlikely(err))
                        goto err;

                skb->dev = fwd;
                _trace_xdp_redirect(dev, xdp_prog, ri->tgt_index);
                generic_xdp_tx(skb, xdp_prog);
                return 0;
        }

        return xdp_do_generic_redirect_map(dev, skb, xdp, xdp_prog, fwd, map_type, map_id, flags);
err:
        _trace_xdp_redirect_err(dev, xdp_prog, ri->tgt_index, err);
        return err;
}

BPF_CALL_2(bpf_xdp_redirect, u32, ifindex, u64, flags)
{
        struct bpf_redirect_info *ri = bpf_net_ctx_get_ri();

        if (unlikely(flags))
                return XDP_ABORTED;

        /* NB! Map type UNSPEC and map_id == INT_MAX (never generated
         * by map_idr) is used for ifindex based XDP redirect.
         */
        ri->tgt_index = ifindex;
        ri->map_id = INT_MAX;
        ri->map_type = BPF_MAP_TYPE_UNSPEC;

        return XDP_REDIRECT;
}

static const struct bpf_func_proto bpf_xdp_redirect_proto = {
        .func           = bpf_xdp_redirect,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_ANYTHING,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_3(bpf_xdp_redirect_map, struct bpf_map *, map, u64, key,
           u64, flags)
{
        return map->ops->map_redirect(map, key, flags);
}

static const struct bpf_func_proto bpf_xdp_redirect_map_proto = {
        .func           = bpf_xdp_redirect_map,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_CONST_MAP_PTR,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_ANYTHING,
};

static unsigned long bpf_skb_copy(void *dst_buff, const void *skb,
                                  unsigned long off, unsigned long len)
{
        void *ptr = skb_header_pointer(skb, off, len, dst_buff);

        if (unlikely(!ptr))
                return len;
        if (ptr != dst_buff)
                memcpy(dst_buff, ptr, len);

        return 0;
}

BPF_CALL_5(bpf_skb_event_output, struct sk_buff *, skb, struct bpf_map *, map,
           u64, flags, void *, meta, u64, meta_size)
{
        u64 skb_size = (flags & BPF_F_CTXLEN_MASK) >> 32;

        if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
                return -EINVAL;
        if (unlikely(!skb || skb_size > skb->len))
                return -EFAULT;

        return bpf_event_output(map, flags, meta, meta_size, skb, skb_size,
                                bpf_skb_copy);
}

static const struct bpf_func_proto bpf_skb_event_output_proto = {
        .func                = bpf_skb_event_output,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE_OR_ZERO,
};

BTF_ID_LIST_SINGLE(bpf_skb_output_btf_ids, struct, sk_buff)

const struct bpf_func_proto bpf_skb_output_proto = {
        .func                = bpf_skb_event_output,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &bpf_skb_output_btf_ids[0],
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE_OR_ZERO,
};

static unsigned short bpf_tunnel_key_af(u64 flags)
{
        return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;
}

BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key *, to,
           u32, size, u64, flags)
{
        const struct ip_tunnel_info *info = skb_tunnel_info(skb);
        u8 compat[sizeof(struct bpf_tunnel_key)];
        void *to_orig = to;
        int err;

        if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6 |
                                         BPF_F_TUNINFO_FLAGS)))) {
                err = -EINVAL;
                goto err_clear;
        }
        if (ip_tunnel_info_af(info) != bpf_tunnel_key_af(flags)) {
                err = -EPROTO;
                goto err_clear;
        }
        if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
                err = -EINVAL;
                switch (size) {
                case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
                case offsetof(struct bpf_tunnel_key, tunnel_label):
                case offsetof(struct bpf_tunnel_key, tunnel_ext):
                        goto set_compat;
                case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
                        /* Fixup deprecated structure layouts here, so we have
                         * a common path later on.
                         */
                        if (ip_tunnel_info_af(info) != AF_INET)
                                goto err_clear;
set_compat:
                        to = (struct bpf_tunnel_key *)compat;
                        break;
                default:
                        goto err_clear;
                }
        }

        to->tunnel_id = be64_to_cpu(info->key.tun_id);
        to->tunnel_tos = info->key.tos;
        to->tunnel_ttl = info->key.ttl;
        if (flags & BPF_F_TUNINFO_FLAGS)
                to->tunnel_flags = ip_tunnel_flags_to_be16(info->key.tun_flags);
        else
                to->tunnel_ext = 0;

        if (flags & BPF_F_TUNINFO_IPV6) {
                memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
                       sizeof(to->remote_ipv6));
                memcpy(to->local_ipv6, &info->key.u.ipv6.dst,
                       sizeof(to->local_ipv6));
                to->tunnel_label = be32_to_cpu(info->key.label);
        } else {
                to->remote_ipv4 = be32_to_cpu(info->key.u.ipv4.src);
                memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
                to->local_ipv4 = be32_to_cpu(info->key.u.ipv4.dst);
                memset(&to->local_ipv6[1], 0, sizeof(__u32) * 3);
                to->tunnel_label = 0;
        }

        if (unlikely(size != sizeof(struct bpf_tunnel_key)))
                memcpy(to_orig, to, size);

        return 0;
err_clear:
        memset(to_orig, 0, size);
        return err;
}

static const struct bpf_func_proto bpf_skb_get_tunnel_key_proto = {
        .func                = bpf_skb_get_tunnel_key,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_3(bpf_skb_get_tunnel_opt, struct sk_buff *, skb, u8 *, to, u32, size)
{
        const struct ip_tunnel_info *info = skb_tunnel_info(skb);
        int err;

        if (unlikely(!info ||
                     !ip_tunnel_is_options_present(info->key.tun_flags))) {
                err = -ENOENT;
                goto err_clear;
        }
        if (unlikely(size < info->options_len)) {
                err = -ENOMEM;
                goto err_clear;
        }

        ip_tunnel_info_opts_get(to, info);
        if (size > info->options_len)
                memset(to + info->options_len, 0, size - info->options_len);

        return info->options_len;
err_clear:
        memset(to, 0, size);
        return err;
}

static const struct bpf_func_proto bpf_skb_get_tunnel_opt_proto = {
        .func                = bpf_skb_get_tunnel_opt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg3_type        = ARG_CONST_SIZE,
};

static struct metadata_dst __percpu *md_dst;

BPF_CALL_4(bpf_skb_set_tunnel_key, struct sk_buff *, skb,
           const struct bpf_tunnel_key *, from, u32, size, u64, flags)
{
        struct metadata_dst *md = this_cpu_ptr(md_dst);
        u8 compat[sizeof(struct bpf_tunnel_key)];
        struct ip_tunnel_info *info;

        if (unlikely(flags & ~(BPF_F_TUNINFO_IPV6 | BPF_F_ZERO_CSUM_TX |
                               BPF_F_DONT_FRAGMENT | BPF_F_SEQ_NUMBER |
                               BPF_F_NO_TUNNEL_KEY)))
                return -EINVAL;
        if (unlikely(size != sizeof(struct bpf_tunnel_key))) {
                switch (size) {
                case offsetof(struct bpf_tunnel_key, local_ipv6[0]):
                case offsetof(struct bpf_tunnel_key, tunnel_label):
                case offsetof(struct bpf_tunnel_key, tunnel_ext):
                case offsetof(struct bpf_tunnel_key, remote_ipv6[1]):
                        /* Fixup deprecated structure layouts here, so we have
                         * a common path later on.
                         */
                        memcpy(compat, from, size);
                        memset(compat + size, 0, sizeof(compat) - size);
                        from = (const struct bpf_tunnel_key *) compat;
                        break;
                default:
                        return -EINVAL;
                }
        }
        if (unlikely((!(flags & BPF_F_TUNINFO_IPV6) && from->tunnel_label) ||
                     from->tunnel_ext))
                return -EINVAL;

        skb_dst_drop(skb);
        dst_hold((struct dst_entry *) md);
        skb_dst_set(skb, (struct dst_entry *) md);

        info = &md->u.tun_info;
        memset(info, 0, sizeof(*info));
        info->mode = IP_TUNNEL_INFO_TX;

        __set_bit(IP_TUNNEL_NOCACHE_BIT, info->key.tun_flags);
        __assign_bit(IP_TUNNEL_DONT_FRAGMENT_BIT, info->key.tun_flags,
                     flags & BPF_F_DONT_FRAGMENT);
        __assign_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags,
                     !(flags & BPF_F_ZERO_CSUM_TX));
        __assign_bit(IP_TUNNEL_SEQ_BIT, info->key.tun_flags,
                     flags & BPF_F_SEQ_NUMBER);
        __assign_bit(IP_TUNNEL_KEY_BIT, info->key.tun_flags,
                     !(flags & BPF_F_NO_TUNNEL_KEY));

        info->key.tun_id = cpu_to_be64(from->tunnel_id);
        info->key.tos = from->tunnel_tos;
        info->key.ttl = from->tunnel_ttl;

        if (flags & BPF_F_TUNINFO_IPV6) {
                info->mode |= IP_TUNNEL_INFO_IPV6;
                memcpy(&info->key.u.ipv6.dst, from->remote_ipv6,
                       sizeof(from->remote_ipv6));
                memcpy(&info->key.u.ipv6.src, from->local_ipv6,
                       sizeof(from->local_ipv6));
                info->key.label = cpu_to_be32(from->tunnel_label) &
                                  IPV6_FLOWLABEL_MASK;
        } else {
                info->key.u.ipv4.dst = cpu_to_be32(from->remote_ipv4);
                info->key.u.ipv4.src = cpu_to_be32(from->local_ipv4);
                info->key.flow_flags = FLOWI_FLAG_ANYSRC;
        }

        return 0;
}

static const struct bpf_func_proto bpf_skb_set_tunnel_key_proto = {
        .func                = bpf_skb_set_tunnel_key,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_3(bpf_skb_set_tunnel_opt, struct sk_buff *, skb,
           const u8 *, from, u32, size)
{
        struct ip_tunnel_info *info = skb_tunnel_info(skb);
        const struct metadata_dst *md = this_cpu_ptr(md_dst);
        IP_TUNNEL_DECLARE_FLAGS(present) = { };

        if (unlikely(info != &md->u.tun_info || (size & (sizeof(u32) - 1))))
                return -EINVAL;
        if (unlikely(size > IP_TUNNEL_OPTS_MAX))
                return -ENOMEM;

        ip_tunnel_set_options_present(present);
        ip_tunnel_info_opts_set(info, from, size, present);

        return 0;
}

static const struct bpf_func_proto bpf_skb_set_tunnel_opt_proto = {
        .func                = bpf_skb_set_tunnel_opt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
};

static const struct bpf_func_proto *
bpf_get_skb_set_tunnel_proto(enum bpf_func_id which)
{
        if (!md_dst) {
                struct metadata_dst __percpu *tmp;

                tmp = metadata_dst_alloc_percpu(IP_TUNNEL_OPTS_MAX,
                                                METADATA_IP_TUNNEL,
                                                GFP_KERNEL);
                if (!tmp)
                        return NULL;
                if (cmpxchg(&md_dst, NULL, tmp))
                        metadata_dst_free_percpu(tmp);
        }

        switch (which) {
        case BPF_FUNC_skb_set_tunnel_key:
                return &bpf_skb_set_tunnel_key_proto;
        case BPF_FUNC_skb_set_tunnel_opt:
                return &bpf_skb_set_tunnel_opt_proto;
        default:
                return NULL;
        }
}

BPF_CALL_3(bpf_skb_under_cgroup, struct sk_buff *, skb, struct bpf_map *, map,
           u32, idx)
{
        struct bpf_array *array = container_of(map, struct bpf_array, map);
        struct cgroup *cgrp;
        struct sock *sk;

        sk = skb_to_full_sk(skb);
        if (!sk || !sk_fullsock(sk))
                return -ENOENT;
        if (unlikely(idx >= array->map.max_entries))
                return -E2BIG;

        cgrp = READ_ONCE(array->ptrs[idx]);
        if (unlikely(!cgrp))
                return -EAGAIN;

        return sk_under_cgroup_hierarchy(sk, cgrp);
}

static const struct bpf_func_proto bpf_skb_under_cgroup_proto = {
        .func                = bpf_skb_under_cgroup,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
};

#ifdef CONFIG_SOCK_CGROUP_DATA
static inline u64 __bpf_sk_cgroup_id(struct sock *sk)
{
        struct cgroup *cgrp;

        sk = sk_to_full_sk(sk);
        if (!sk || !sk_fullsock(sk))
                return 0;

        cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        return cgroup_id(cgrp);
}

BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb)
{
        return __bpf_sk_cgroup_id(skb->sk);
}

static const struct bpf_func_proto bpf_skb_cgroup_id_proto = {
        .func           = bpf_skb_cgroup_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

static inline u64 __bpf_sk_ancestor_cgroup_id(struct sock *sk,
                                              int ancestor_level)
{
        struct cgroup *ancestor;
        struct cgroup *cgrp;

        sk = sk_to_full_sk(sk);
        if (!sk || !sk_fullsock(sk))
                return 0;

        cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
        ancestor = cgroup_ancestor(cgrp, ancestor_level);
        if (!ancestor)
                return 0;

        return cgroup_id(ancestor);
}

BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int,
           ancestor_level)
{
        return __bpf_sk_ancestor_cgroup_id(skb->sk, ancestor_level);
}

static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = {
        .func           = bpf_skb_ancestor_cgroup_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
};

BPF_CALL_1(bpf_sk_cgroup_id, struct sock *, sk)
{
        return __bpf_sk_cgroup_id(sk);
}

static const struct bpf_func_proto bpf_sk_cgroup_id_proto = {
        .func           = bpf_sk_cgroup_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
};

BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level)
{
        return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level);
}

static const struct bpf_func_proto bpf_sk_ancestor_cgroup_id_proto = {
        .func           = bpf_sk_ancestor_cgroup_id,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type      = ARG_ANYTHING,
};
#endif

static unsigned long bpf_xdp_copy(void *dst, const void *ctx,
                                  unsigned long off, unsigned long len)
{
        struct xdp_buff *xdp = (struct xdp_buff *)ctx;

        bpf_xdp_copy_buf(xdp, off, dst, len, false);
        return 0;
}

BPF_CALL_5(bpf_xdp_event_output, struct xdp_buff *, xdp, struct bpf_map *, map,
           u64, flags, void *, meta, u64, meta_size)
{
        u64 xdp_size = (flags & BPF_F_CTXLEN_MASK) >> 32;

        if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
                return -EINVAL;

        if (unlikely(!xdp || xdp_size > xdp_get_buff_len(xdp)))
                return -EFAULT;

        return bpf_event_output(map, flags, meta, meta_size, xdp,
                                xdp_size, bpf_xdp_copy);
}

static const struct bpf_func_proto bpf_xdp_event_output_proto = {
        .func                = bpf_xdp_event_output,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE_OR_ZERO,
};

BTF_ID_LIST_SINGLE(bpf_xdp_output_btf_ids, struct, xdp_buff)

const struct bpf_func_proto bpf_xdp_output_proto = {
        .func                = bpf_xdp_event_output,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &bpf_xdp_output_btf_ids[0],
        .arg2_type        = ARG_CONST_MAP_PTR,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE_OR_ZERO,
};

BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb)
{
        return skb->sk ? __sock_gen_cookie(skb->sk) : 0;
}

static const struct bpf_func_proto bpf_get_socket_cookie_proto = {
        .func           = bpf_get_socket_cookie,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
{
        return __sock_gen_cookie(ctx->sk);
}

static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = {
        .func                = bpf_get_socket_cookie_sock_addr,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx)
{
        return __sock_gen_cookie(ctx);
}

static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = {
        .func                = bpf_get_socket_cookie_sock,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

BPF_CALL_1(bpf_get_socket_ptr_cookie, struct sock *, sk)
{
        return sk ? sock_gen_cookie(sk) : 0;
}

const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto = {
        .func                = bpf_get_socket_ptr_cookie,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON | PTR_MAYBE_NULL,
};

BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
{
        return __sock_gen_cookie(ctx->sk);
}

static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = {
        .func                = bpf_get_socket_cookie_sock_ops,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
};

static u64 __bpf_get_netns_cookie(struct sock *sk)
{
        const struct net *net = sk ? sock_net(sk) : &init_net;

        return net->net_cookie;
}

BPF_CALL_1(bpf_get_netns_cookie, struct sk_buff *, skb)
{
        return __bpf_get_netns_cookie(skb && skb->sk ? skb->sk : NULL);
}

static const struct bpf_func_proto bpf_get_netns_cookie_proto = {
        .func           = bpf_get_netns_cookie,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX_OR_NULL,
};

BPF_CALL_1(bpf_get_netns_cookie_sock, struct sock *, ctx)
{
        return __bpf_get_netns_cookie(ctx);
}

static const struct bpf_func_proto bpf_get_netns_cookie_sock_proto = {
        .func                = bpf_get_netns_cookie_sock,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX_OR_NULL,
};

BPF_CALL_1(bpf_get_netns_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
{
        return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
}

static const struct bpf_func_proto bpf_get_netns_cookie_sock_addr_proto = {
        .func                = bpf_get_netns_cookie_sock_addr,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX_OR_NULL,
};

BPF_CALL_1(bpf_get_netns_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
{
        return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
}

static const struct bpf_func_proto bpf_get_netns_cookie_sock_ops_proto = {
        .func                = bpf_get_netns_cookie_sock_ops,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX_OR_NULL,
};

BPF_CALL_1(bpf_get_netns_cookie_sk_msg, struct sk_msg *, ctx)
{
        return __bpf_get_netns_cookie(ctx ? ctx->sk : NULL);
}

static const struct bpf_func_proto bpf_get_netns_cookie_sk_msg_proto = {
        .func                = bpf_get_netns_cookie_sk_msg,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX_OR_NULL,
};

BPF_CALL_1(bpf_get_socket_uid, struct sk_buff *, skb)
{
        struct sock *sk = sk_to_full_sk(skb->sk);
        kuid_t kuid;

        if (!sk || !sk_fullsock(sk))
                return overflowuid;
        kuid = sock_net_uid(sock_net(sk), sk);
        return from_kuid_munged(sock_net(sk)->user_ns, kuid);
}

static const struct bpf_func_proto bpf_get_socket_uid_proto = {
        .func           = bpf_get_socket_uid,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

static int sk_bpf_set_get_cb_flags(struct sock *sk, char *optval, bool getopt)
{
        u32 sk_bpf_cb_flags;

        if (getopt) {
                *(u32 *)optval = sk->sk_bpf_cb_flags;
                return 0;
        }

        sk_bpf_cb_flags = *(u32 *)optval;

        if (sk_bpf_cb_flags & ~SK_BPF_CB_MASK)
                return -EINVAL;

        sk->sk_bpf_cb_flags = sk_bpf_cb_flags;

        return 0;
}

static int sol_socket_sockopt(struct sock *sk, int optname,
                              char *optval, int *optlen,
                              bool getopt)
{
        switch (optname) {
        case SO_REUSEADDR:
        case SO_SNDBUF:
        case SO_RCVBUF:
        case SO_KEEPALIVE:
        case SO_PRIORITY:
        case SO_REUSEPORT:
        case SO_RCVLOWAT:
        case SO_MARK:
        case SO_MAX_PACING_RATE:
        case SO_BINDTOIFINDEX:
        case SO_TXREHASH:
        case SK_BPF_CB_FLAGS:
                if (*optlen != sizeof(int))
                        return -EINVAL;
                break;
        case SO_BINDTODEVICE:
                break;
        default:
                return -EINVAL;
        }

        if (optname == SK_BPF_CB_FLAGS)
                return sk_bpf_set_get_cb_flags(sk, optval, getopt);

        if (getopt) {
                if (optname == SO_BINDTODEVICE)
                        return -EINVAL;
                return sk_getsockopt(sk, SOL_SOCKET, optname,
                                     KERNEL_SOCKPTR(optval),
                                     KERNEL_SOCKPTR(optlen));
        }

        return sk_setsockopt(sk, SOL_SOCKET, optname,
                             KERNEL_SOCKPTR(optval), *optlen);
}

static int bpf_sol_tcp_getsockopt(struct sock *sk, int optname,
                                  char *optval, int optlen)
{
        if (optlen != sizeof(int))
                return -EINVAL;

        switch (optname) {
        case TCP_BPF_SOCK_OPS_CB_FLAGS: {
                int cb_flags = tcp_sk(sk)->bpf_sock_ops_cb_flags;

                memcpy(optval, &cb_flags, optlen);
                break;
        }
        case TCP_BPF_RTO_MIN: {
                int rto_min_us = jiffies_to_usecs(inet_csk(sk)->icsk_rto_min);

                memcpy(optval, &rto_min_us, optlen);
                break;
        }
        case TCP_BPF_DELACK_MAX: {
                int delack_max_us = jiffies_to_usecs(inet_csk(sk)->icsk_delack_max);

                memcpy(optval, &delack_max_us, optlen);
                break;
        }
        default:
                return -EINVAL;
        }

        return 0;
}

static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
                                  char *optval, int optlen)
{
        struct tcp_sock *tp = tcp_sk(sk);
        unsigned long timeout;
        int val;

        if (optlen != sizeof(int))
                return -EINVAL;

        val = *(int *)optval;

        /* Only some options are supported */
        switch (optname) {
        case TCP_BPF_IW:
                if (val <= 0 || tp->data_segs_out > tp->syn_data)
                        return -EINVAL;
                tcp_snd_cwnd_set(tp, val);
                break;
        case TCP_BPF_SNDCWND_CLAMP:
                if (val <= 0)
                        return -EINVAL;
                tp->snd_cwnd_clamp = val;
                tp->snd_ssthresh = val;
                break;
        case TCP_BPF_DELACK_MAX:
                timeout = usecs_to_jiffies(val);
                if (timeout > TCP_DELACK_MAX ||
                    timeout < TCP_TIMEOUT_MIN)
                        return -EINVAL;
                inet_csk(sk)->icsk_delack_max = timeout;
                break;
        case TCP_BPF_RTO_MIN:
                timeout = usecs_to_jiffies(val);
                if (timeout > TCP_RTO_MIN ||
                    timeout < TCP_TIMEOUT_MIN)
                        return -EINVAL;
                inet_csk(sk)->icsk_rto_min = timeout;
                break;
        case TCP_BPF_SOCK_OPS_CB_FLAGS:
                if (val & ~(BPF_SOCK_OPS_ALL_CB_FLAGS))
                        return -EINVAL;
                tp->bpf_sock_ops_cb_flags = val;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static int sol_tcp_sockopt_congestion(struct sock *sk, char *optval,
                                      int *optlen, bool getopt)
{
        struct tcp_sock *tp;
        int ret;

        if (*optlen < 2)
                return -EINVAL;

        if (getopt) {
                if (!inet_csk(sk)->icsk_ca_ops)
                        return -EINVAL;
                /* BPF expects NULL-terminated tcp-cc string */
                optval[--(*optlen)] = '\0';
                return do_tcp_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
                                         KERNEL_SOCKPTR(optval),
                                         KERNEL_SOCKPTR(optlen));
        }

        /* "cdg" is the only cc that alloc a ptr
         * in inet_csk_ca area.  The bpf-tcp-cc may
         * overwrite this ptr after switching to cdg.
         */
        if (*optlen >= sizeof("cdg") - 1 && !strncmp("cdg", optval, *optlen))
                return -ENOTSUPP;

        /* It stops this looping
         *
         * .init => bpf_setsockopt(tcp_cc) => .init =>
         * bpf_setsockopt(tcp_cc)" => .init => ....
         *
         * The second bpf_setsockopt(tcp_cc) is not allowed
         * in order to break the loop when both .init
         * are the same bpf prog.
         *
         * This applies even the second bpf_setsockopt(tcp_cc)
         * does not cause a loop.  This limits only the first
         * '.init' can call bpf_setsockopt(TCP_CONGESTION) to
         * pick a fallback cc (eg. peer does not support ECN)
         * and the second '.init' cannot fallback to
         * another.
         */
        tp = tcp_sk(sk);
        if (tp->bpf_chg_cc_inprogress)
                return -EBUSY;

        tp->bpf_chg_cc_inprogress = 1;
        ret = do_tcp_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
                                KERNEL_SOCKPTR(optval), *optlen);
        tp->bpf_chg_cc_inprogress = 0;
        return ret;
}

static int sol_tcp_sockopt(struct sock *sk, int optname,
                           char *optval, int *optlen,
                           bool getopt)
{
        if (sk->sk_protocol != IPPROTO_TCP)
                return -EINVAL;

        switch (optname) {
        case TCP_NODELAY:
        case TCP_MAXSEG:
        case TCP_KEEPIDLE:
        case TCP_KEEPINTVL:
        case TCP_KEEPCNT:
        case TCP_SYNCNT:
        case TCP_WINDOW_CLAMP:
        case TCP_THIN_LINEAR_TIMEOUTS:
        case TCP_USER_TIMEOUT:
        case TCP_NOTSENT_LOWAT:
        case TCP_SAVE_SYN:
        case TCP_RTO_MAX_MS:
                if (*optlen != sizeof(int))
                        return -EINVAL;
                break;
        case TCP_CONGESTION:
                return sol_tcp_sockopt_congestion(sk, optval, optlen, getopt);
        case TCP_SAVED_SYN:
                if (*optlen < 1)
                        return -EINVAL;
                break;
        default:
                if (getopt)
                        return bpf_sol_tcp_getsockopt(sk, optname, optval, *optlen);
                return bpf_sol_tcp_setsockopt(sk, optname, optval, *optlen);
        }

        if (getopt) {
                if (optname == TCP_SAVED_SYN) {
                        struct tcp_sock *tp = tcp_sk(sk);

                        if (!tp->saved_syn ||
                            *optlen > tcp_saved_syn_len(tp->saved_syn))
                                return -EINVAL;
                        memcpy(optval, tp->saved_syn->data, *optlen);
                        /* It cannot free tp->saved_syn here because it
                         * does not know if the user space still needs it.
                         */
                        return 0;
                }

                return do_tcp_getsockopt(sk, SOL_TCP, optname,
                                         KERNEL_SOCKPTR(optval),
                                         KERNEL_SOCKPTR(optlen));
        }

        return do_tcp_setsockopt(sk, SOL_TCP, optname,
                                 KERNEL_SOCKPTR(optval), *optlen);
}

static int sol_ip_sockopt(struct sock *sk, int optname,
                          char *optval, int *optlen,
                          bool getopt)
{
        if (sk->sk_family != AF_INET)
                return -EINVAL;

        switch (optname) {
        case IP_TOS:
                if (*optlen != sizeof(int))
                        return -EINVAL;
                break;
        default:
                return -EINVAL;
        }

        if (getopt)
                return do_ip_getsockopt(sk, SOL_IP, optname,
                                        KERNEL_SOCKPTR(optval),
                                        KERNEL_SOCKPTR(optlen));

        return do_ip_setsockopt(sk, SOL_IP, optname,
                                KERNEL_SOCKPTR(optval), *optlen);
}

static int sol_ipv6_sockopt(struct sock *sk, int optname,
                            char *optval, int *optlen,
                            bool getopt)
{
        if (sk->sk_family != AF_INET6)
                return -EINVAL;

        switch (optname) {
        case IPV6_TCLASS:
        case IPV6_AUTOFLOWLABEL:
                if (*optlen != sizeof(int))
                        return -EINVAL;
                break;
        default:
                return -EINVAL;
        }

        if (getopt)
                return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname,
                                                      KERNEL_SOCKPTR(optval),
                                                      KERNEL_SOCKPTR(optlen));

        return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname,
                                              KERNEL_SOCKPTR(optval), *optlen);
}

static int __bpf_setsockopt(struct sock *sk, int level, int optname,
                            char *optval, int optlen)
{
        if (!sk_fullsock(sk))
                return -EINVAL;

        if (level == SOL_SOCKET)
                return sol_socket_sockopt(sk, optname, optval, &optlen, false);
        else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
                return sol_ip_sockopt(sk, optname, optval, &optlen, false);
        else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
                return sol_ipv6_sockopt(sk, optname, optval, &optlen, false);
        else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
                return sol_tcp_sockopt(sk, optname, optval, &optlen, false);

        return -EINVAL;
}

static bool is_locked_tcp_sock_ops(struct bpf_sock_ops_kern *bpf_sock)
{
        return bpf_sock->op <= BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
}

static int _bpf_setsockopt(struct sock *sk, int level, int optname,
                           char *optval, int optlen)
{
        if (sk_fullsock(sk))
                sock_owned_by_me(sk);
        return __bpf_setsockopt(sk, level, optname, optval, optlen);
}

static int __bpf_getsockopt(struct sock *sk, int level, int optname,
                            char *optval, int optlen)
{
        int err, saved_optlen = optlen;

        if (!sk_fullsock(sk)) {
                err = -EINVAL;
                goto done;
        }

        if (level == SOL_SOCKET)
                err = sol_socket_sockopt(sk, optname, optval, &optlen, true);
        else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
                err = sol_tcp_sockopt(sk, optname, optval, &optlen, true);
        else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
                err = sol_ip_sockopt(sk, optname, optval, &optlen, true);
        else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
                err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true);
        else
                err = -EINVAL;

done:
        if (err)
                optlen = 0;
        if (optlen < saved_optlen)
                memset(optval + optlen, 0, saved_optlen - optlen);
        return err;
}

static int _bpf_getsockopt(struct sock *sk, int level, int optname,
                           char *optval, int optlen)
{
        if (sk_fullsock(sk))
                sock_owned_by_me(sk);
        return __bpf_getsockopt(sk, level, optname, optval, optlen);
}

BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
{
        return _bpf_setsockopt(sk, level, optname, optval, optlen);
}

const struct bpf_func_proto bpf_sk_setsockopt_proto = {
        .func                = bpf_sk_setsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_sk_getsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
{
        return _bpf_getsockopt(sk, level, optname, optval, optlen);
}

const struct bpf_func_proto bpf_sk_getsockopt_proto = {
        .func                = bpf_sk_getsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_unlocked_sk_setsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
{
        return __bpf_setsockopt(sk, level, optname, optval, optlen);
}

const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto = {
        .func                = bpf_unlocked_sk_setsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_unlocked_sk_getsockopt, struct sock *, sk, int, level,
           int, optname, char *, optval, int, optlen)
{
        return __bpf_getsockopt(sk, level, optname, optval, optlen);
}

const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto = {
        .func                = bpf_unlocked_sk_getsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_sock_addr_setsockopt, struct bpf_sock_addr_kern *, ctx,
           int, level, int, optname, char *, optval, int, optlen)
{
        return _bpf_setsockopt(ctx->sk, level, optname, optval, optlen);
}

static const struct bpf_func_proto bpf_sock_addr_setsockopt_proto = {
        .func                = bpf_sock_addr_setsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_sock_addr_getsockopt, struct bpf_sock_addr_kern *, ctx,
           int, level, int, optname, char *, optval, int, optlen)
{
        return _bpf_getsockopt(ctx->sk, level, optname, optval, optlen);
}

static const struct bpf_func_proto bpf_sock_addr_getsockopt_proto = {
        .func                = bpf_sock_addr_getsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_sock_ops_setsockopt, struct bpf_sock_ops_kern *, bpf_sock,
           int, level, int, optname, char *, optval, int, optlen)
{
        if (!is_locked_tcp_sock_ops(bpf_sock))
                return -EOPNOTSUPP;

        return _bpf_setsockopt(bpf_sock->sk, level, optname, optval, optlen);
}

static const struct bpf_func_proto bpf_sock_ops_setsockopt_proto = {
        .func                = bpf_sock_ops_setsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock,
                                int optname, const u8 **start)
{
        struct sk_buff *syn_skb = bpf_sock->syn_skb;
        const u8 *hdr_start;
        int ret;

        if (syn_skb) {
                /* sk is a request_sock here */

                if (optname == TCP_BPF_SYN) {
                        hdr_start = syn_skb->data;
                        ret = tcp_hdrlen(syn_skb);
                } else if (optname == TCP_BPF_SYN_IP) {
                        hdr_start = skb_network_header(syn_skb);
                        ret = skb_network_header_len(syn_skb) +
                                tcp_hdrlen(syn_skb);
                } else {
                        /* optname == TCP_BPF_SYN_MAC */
                        hdr_start = skb_mac_header(syn_skb);
                        ret = skb_mac_header_len(syn_skb) +
                                skb_network_header_len(syn_skb) +
                                tcp_hdrlen(syn_skb);
                }
        } else {
                struct sock *sk = bpf_sock->sk;
                struct saved_syn *saved_syn;

                if (sk->sk_state == TCP_NEW_SYN_RECV)
                        /* synack retransmit. bpf_sock->syn_skb will
                         * not be available.  It has to resort to
                         * saved_syn (if it is saved).
                         */
                        saved_syn = inet_reqsk(sk)->saved_syn;
                else
                        saved_syn = tcp_sk(sk)->saved_syn;

                if (!saved_syn)
                        return -ENOENT;

                if (optname == TCP_BPF_SYN) {
                        hdr_start = saved_syn->data +
                                saved_syn->mac_hdrlen +
                                saved_syn->network_hdrlen;
                        ret = saved_syn->tcp_hdrlen;
                } else if (optname == TCP_BPF_SYN_IP) {
                        hdr_start = saved_syn->data +
                                saved_syn->mac_hdrlen;
                        ret = saved_syn->network_hdrlen +
                                saved_syn->tcp_hdrlen;
                } else {
                        /* optname == TCP_BPF_SYN_MAC */

                        /* TCP_SAVE_SYN may not have saved the mac hdr */
                        if (!saved_syn->mac_hdrlen)
                                return -ENOENT;

                        hdr_start = saved_syn->data;
                        ret = saved_syn->mac_hdrlen +
                                saved_syn->network_hdrlen +
                                saved_syn->tcp_hdrlen;
                }
        }

        *start = hdr_start;
        return ret;
}

BPF_CALL_5(bpf_sock_ops_getsockopt, struct bpf_sock_ops_kern *, bpf_sock,
           int, level, int, optname, char *, optval, int, optlen)
{
        if (!is_locked_tcp_sock_ops(bpf_sock))
                return -EOPNOTSUPP;

        if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP &&
            optname >= TCP_BPF_SYN && optname <= TCP_BPF_SYN_MAC) {
                int ret, copy_len = 0;
                const u8 *start;

                ret = bpf_sock_ops_get_syn(bpf_sock, optname, &start);
                if (ret > 0) {
                        copy_len = ret;
                        if (optlen < copy_len) {
                                copy_len = optlen;
                                ret = -ENOSPC;
                        }

                        memcpy(optval, start, copy_len);
                }

                /* Zero out unused buffer at the end */
                memset(optval + copy_len, 0, optlen - copy_len);

                return ret;
        }

        return _bpf_getsockopt(bpf_sock->sk, level, optname, optval, optlen);
}

static const struct bpf_func_proto bpf_sock_ops_getsockopt_proto = {
        .func                = bpf_sock_ops_getsockopt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
        .arg4_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock,
           int, argval)
{
        struct sock *sk = bpf_sock->sk;
        int val = argval & BPF_SOCK_OPS_ALL_CB_FLAGS;

        if (!is_locked_tcp_sock_ops(bpf_sock))
                return -EOPNOTSUPP;

        if (!IS_ENABLED(CONFIG_INET) || !sk_fullsock(sk))
                return -EINVAL;

        tcp_sk(sk)->bpf_sock_ops_cb_flags = val;

        return argval & (~BPF_SOCK_OPS_ALL_CB_FLAGS);
}

static const struct bpf_func_proto bpf_sock_ops_cb_flags_set_proto = {
        .func                = bpf_sock_ops_cb_flags_set,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
};

const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly;
EXPORT_SYMBOL_GPL(ipv6_bpf_stub);

BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr,
           int, addr_len)
{
#ifdef CONFIG_INET
        struct sock *sk = ctx->sk;
        u32 flags = BIND_FROM_BPF;
        int err;

        err = -EINVAL;
        if (addr_len < offsetofend(struct sockaddr, sa_family))
                return err;
        if (addr->sa_family == AF_INET) {
                if (addr_len < sizeof(struct sockaddr_in))
                        return err;
                if (((struct sockaddr_in *)addr)->sin_port == htons(0))
                        flags |= BIND_FORCE_ADDRESS_NO_PORT;
                return __inet_bind(sk, addr, addr_len, flags);
#if IS_ENABLED(CONFIG_IPV6)
        } else if (addr->sa_family == AF_INET6) {
                if (addr_len < SIN6_LEN_RFC2133)
                        return err;
                if (((struct sockaddr_in6 *)addr)->sin6_port == htons(0))
                        flags |= BIND_FORCE_ADDRESS_NO_PORT;
                /* ipv6_bpf_stub cannot be NULL, since it's called from
                 * bpf_cgroup_inet6_connect hook and ipv6 is already loaded
                 */
                return ipv6_bpf_stub->inet6_bind(sk, addr, addr_len, flags);
#endif /* CONFIG_IPV6 */
        }
#endif /* CONFIG_INET */

        return -EAFNOSUPPORT;
}

static const struct bpf_func_proto bpf_bind_proto = {
        .func                = bpf_bind,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
};

#ifdef CONFIG_XFRM

#if (IS_BUILTIN(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
    (IS_MODULE(CONFIG_XFRM_INTERFACE) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))

struct metadata_dst __percpu *xfrm_bpf_md_dst;
EXPORT_SYMBOL_GPL(xfrm_bpf_md_dst);

#endif

BPF_CALL_5(bpf_skb_get_xfrm_state, struct sk_buff *, skb, u32, index,
           struct bpf_xfrm_state *, to, u32, size, u64, flags)
{
        const struct sec_path *sp = skb_sec_path(skb);
        const struct xfrm_state *x;

        if (!sp || unlikely(index >= sp->len || flags))
                goto err_clear;

        x = sp->xvec[index];

        if (unlikely(size != sizeof(struct bpf_xfrm_state)))
                goto err_clear;

        to->reqid = x->props.reqid;
        to->spi = x->id.spi;
        to->family = x->props.family;
        to->ext = 0;

        if (to->family == AF_INET6) {
                memcpy(to->remote_ipv6, x->props.saddr.a6,
                       sizeof(to->remote_ipv6));
        } else {
                to->remote_ipv4 = x->props.saddr.a4;
                memset(&to->remote_ipv6[1], 0, sizeof(__u32) * 3);
        }

        return 0;
err_clear:
        memset(to, 0, size);
        return -EINVAL;
}

static const struct bpf_func_proto bpf_skb_get_xfrm_state_proto = {
        .func                = bpf_skb_get_xfrm_state,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
        .arg5_type        = ARG_ANYTHING,
};
#endif

#if IS_ENABLED(CONFIG_INET) || IS_ENABLED(CONFIG_IPV6)
static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params, u32 mtu)
{
        params->h_vlan_TCI = 0;
        params->h_vlan_proto = 0;
        if (mtu)
                params->mtu_result = mtu; /* union with tot_len */

        return 0;
}
#endif

#if IS_ENABLED(CONFIG_INET)
static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
                               u32 flags, bool check_mtu)
{
        struct fib_nh_common *nhc;
        struct in_device *in_dev;
        struct neighbour *neigh;
        struct net_device *dev;
        struct fib_result res;
        struct flowi4 fl4;
        u32 mtu = 0;
        int err;

        dev = dev_get_by_index_rcu(net, params->ifindex);
        if (unlikely(!dev))
                return -ENODEV;

        /* verify forwarding is enabled on this interface */
        in_dev = __in_dev_get_rcu(dev);
        if (unlikely(!in_dev || !IN_DEV_FORWARD(in_dev)))
                return BPF_FIB_LKUP_RET_FWD_DISABLED;

        if (flags & BPF_FIB_LOOKUP_OUTPUT) {
                fl4.flowi4_iif = 1;
                fl4.flowi4_oif = params->ifindex;
        } else {
                fl4.flowi4_iif = params->ifindex;
                fl4.flowi4_oif = 0;
        }
        fl4.flowi4_tos = params->tos & INET_DSCP_MASK;
        fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
        fl4.flowi4_flags = 0;

        fl4.flowi4_proto = params->l4_protocol;
        fl4.daddr = params->ipv4_dst;
        fl4.saddr = params->ipv4_src;
        fl4.fl4_sport = params->sport;
        fl4.fl4_dport = params->dport;
        fl4.flowi4_multipath_hash = 0;

        if (flags & BPF_FIB_LOOKUP_DIRECT) {
                u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
                struct fib_table *tb;

                if (flags & BPF_FIB_LOOKUP_TBID) {
                        tbid = params->tbid;
                        /* zero out for vlan output */
                        params->tbid = 0;
                }

                tb = fib_get_table(net, tbid);
                if (unlikely(!tb))
                        return BPF_FIB_LKUP_RET_NOT_FWDED;

                err = fib_table_lookup(tb, &fl4, &res, FIB_LOOKUP_NOREF);
        } else {
                if (flags & BPF_FIB_LOOKUP_MARK)
                        fl4.flowi4_mark = params->mark;
                else
                        fl4.flowi4_mark = 0;
                fl4.flowi4_secid = 0;
                fl4.flowi4_tun_key.tun_id = 0;
                fl4.flowi4_uid = sock_net_uid(net, NULL);

                err = fib_lookup(net, &fl4, &res, FIB_LOOKUP_NOREF);
        }

        if (err) {
                /* map fib lookup errors to RTN_ type */
                if (err == -EINVAL)
                        return BPF_FIB_LKUP_RET_BLACKHOLE;
                if (err == -EHOSTUNREACH)
                        return BPF_FIB_LKUP_RET_UNREACHABLE;
                if (err == -EACCES)
                        return BPF_FIB_LKUP_RET_PROHIBIT;

                return BPF_FIB_LKUP_RET_NOT_FWDED;
        }

        if (res.type != RTN_UNICAST)
                return BPF_FIB_LKUP_RET_NOT_FWDED;

        if (fib_info_num_path(res.fi) > 1)
                fib_select_path(net, &res, &fl4, NULL);

        if (check_mtu) {
                mtu = ip_mtu_from_fib_result(&res, params->ipv4_dst);
                if (params->tot_len > mtu) {
                        params->mtu_result = mtu; /* union with tot_len */
                        return BPF_FIB_LKUP_RET_FRAG_NEEDED;
                }
        }

        nhc = res.nhc;

        /* do not handle lwt encaps right now */
        if (nhc->nhc_lwtstate)
                return BPF_FIB_LKUP_RET_UNSUPP_LWT;

        dev = nhc->nhc_dev;

        params->rt_metric = res.fi->fib_priority;
        params->ifindex = dev->ifindex;

        if (flags & BPF_FIB_LOOKUP_SRC)
                params->ipv4_src = fib_result_prefsrc(net, &res);

        /* xdp and cls_bpf programs are run in RCU-bh so
         * rcu_read_lock_bh is not needed here
         */
        if (likely(nhc->nhc_gw_family != AF_INET6)) {
                if (nhc->nhc_gw_family)
                        params->ipv4_dst = nhc->nhc_gw.ipv4;
        } else {
                struct in6_addr *dst = (struct in6_addr *)params->ipv6_dst;

                params->family = AF_INET6;
                *dst = nhc->nhc_gw.ipv6;
        }

        if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
                goto set_fwd_params;

        if (likely(nhc->nhc_gw_family != AF_INET6))
                neigh = __ipv4_neigh_lookup_noref(dev,
                                                  (__force u32)params->ipv4_dst);
        else
                neigh = __ipv6_neigh_lookup_noref_stub(dev, params->ipv6_dst);

        if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
                return BPF_FIB_LKUP_RET_NO_NEIGH;
        memcpy(params->dmac, neigh->ha, ETH_ALEN);
        memcpy(params->smac, dev->dev_addr, ETH_ALEN);

set_fwd_params:
        return bpf_fib_set_fwd_params(params, mtu);
}
#endif

#if IS_ENABLED(CONFIG_IPV6)
static int bpf_ipv6_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
                               u32 flags, bool check_mtu)
{
        struct in6_addr *src = (struct in6_addr *) params->ipv6_src;
        struct in6_addr *dst = (struct in6_addr *) params->ipv6_dst;
        struct fib6_result res = {};
        struct neighbour *neigh;
        struct net_device *dev;
        struct inet6_dev *idev;
        struct flowi6 fl6;
        int strict = 0;
        int oif, err;
        u32 mtu = 0;

        /* link local addresses are never forwarded */
        if (rt6_need_strict(dst) || rt6_need_strict(src))
                return BPF_FIB_LKUP_RET_NOT_FWDED;

        dev = dev_get_by_index_rcu(net, params->ifindex);
        if (unlikely(!dev))
                return -ENODEV;

        idev = __in6_dev_get_safely(dev);
        if (unlikely(!idev || !READ_ONCE(idev->cnf.forwarding)))
                return BPF_FIB_LKUP_RET_FWD_DISABLED;

        if (flags & BPF_FIB_LOOKUP_OUTPUT) {
                fl6.flowi6_iif = 1;
                oif = fl6.flowi6_oif = params->ifindex;
        } else {
                oif = fl6.flowi6_iif = params->ifindex;
                fl6.flowi6_oif = 0;
                strict = RT6_LOOKUP_F_HAS_SADDR;
        }
        fl6.flowlabel = params->flowinfo;
        fl6.flowi6_scope = 0;
        fl6.flowi6_flags = 0;
        fl6.mp_hash = 0;

        fl6.flowi6_proto = params->l4_protocol;
        fl6.daddr = *dst;
        fl6.saddr = *src;
        fl6.fl6_sport = params->sport;
        fl6.fl6_dport = params->dport;

        if (flags & BPF_FIB_LOOKUP_DIRECT) {
                u32 tbid = l3mdev_fib_table_rcu(dev) ? : RT_TABLE_MAIN;
                struct fib6_table *tb;

                if (flags & BPF_FIB_LOOKUP_TBID) {
                        tbid = params->tbid;
                        /* zero out for vlan output */
                        params->tbid = 0;
                }

                tb = ipv6_stub->fib6_get_table(net, tbid);
                if (unlikely(!tb))
                        return BPF_FIB_LKUP_RET_NOT_FWDED;

                err = ipv6_stub->fib6_table_lookup(net, tb, oif, &fl6, &res,
                                                   strict);
        } else {
                if (flags & BPF_FIB_LOOKUP_MARK)
                        fl6.flowi6_mark = params->mark;
                else
                        fl6.flowi6_mark = 0;
                fl6.flowi6_secid = 0;
                fl6.flowi6_tun_key.tun_id = 0;
                fl6.flowi6_uid = sock_net_uid(net, NULL);

                err = ipv6_stub->fib6_lookup(net, oif, &fl6, &res, strict);
        }

        if (unlikely(err || IS_ERR_OR_NULL(res.f6i) ||
                     res.f6i == net->ipv6.fib6_null_entry))
                return BPF_FIB_LKUP_RET_NOT_FWDED;

        switch (res.fib6_type) {
        /* only unicast is forwarded */
        case RTN_UNICAST:
                break;
        case RTN_BLACKHOLE:
                return BPF_FIB_LKUP_RET_BLACKHOLE;
        case RTN_UNREACHABLE:
                return BPF_FIB_LKUP_RET_UNREACHABLE;
        case RTN_PROHIBIT:
                return BPF_FIB_LKUP_RET_PROHIBIT;
        default:
                return BPF_FIB_LKUP_RET_NOT_FWDED;
        }

        ipv6_stub->fib6_select_path(net, &res, &fl6, fl6.flowi6_oif,
                                    fl6.flowi6_oif != 0, NULL, strict);

        if (check_mtu) {
                mtu = ipv6_stub->ip6_mtu_from_fib6(&res, dst, src);
                if (params->tot_len > mtu) {
                        params->mtu_result = mtu; /* union with tot_len */
                        return BPF_FIB_LKUP_RET_FRAG_NEEDED;
                }
        }

        if (res.nh->fib_nh_lws)
                return BPF_FIB_LKUP_RET_UNSUPP_LWT;

        if (res.nh->fib_nh_gw_family)
                *dst = res.nh->fib_nh_gw6;

        dev = res.nh->fib_nh_dev;
        params->rt_metric = res.f6i->fib6_metric;
        params->ifindex = dev->ifindex;

        if (flags & BPF_FIB_LOOKUP_SRC) {
                if (res.f6i->fib6_prefsrc.plen) {
                        *src = res.f6i->fib6_prefsrc.addr;
                } else {
                        err = ipv6_bpf_stub->ipv6_dev_get_saddr(net, dev,
                                                                &fl6.daddr, 0,
                                                                src);
                        if (err)
                                return BPF_FIB_LKUP_RET_NO_SRC_ADDR;
                }
        }

        if (flags & BPF_FIB_LOOKUP_SKIP_NEIGH)
                goto set_fwd_params;

        /* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is
         * not needed here.
         */
        neigh = __ipv6_neigh_lookup_noref_stub(dev, dst);
        if (!neigh || !(READ_ONCE(neigh->nud_state) & NUD_VALID))
                return BPF_FIB_LKUP_RET_NO_NEIGH;
        memcpy(params->dmac, neigh->ha, ETH_ALEN);
        memcpy(params->smac, dev->dev_addr, ETH_ALEN);

set_fwd_params:
        return bpf_fib_set_fwd_params(params, mtu);
}
#endif

#define BPF_FIB_LOOKUP_MASK (BPF_FIB_LOOKUP_DIRECT | BPF_FIB_LOOKUP_OUTPUT | \
                             BPF_FIB_LOOKUP_SKIP_NEIGH | BPF_FIB_LOOKUP_TBID | \
                             BPF_FIB_LOOKUP_SRC | BPF_FIB_LOOKUP_MARK)

BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx,
           struct bpf_fib_lookup *, params, int, plen, u32, flags)
{
        if (plen < sizeof(*params))
                return -EINVAL;

        if (flags & ~BPF_FIB_LOOKUP_MASK)
                return -EINVAL;

        switch (params->family) {
#if IS_ENABLED(CONFIG_INET)
        case AF_INET:
                return bpf_ipv4_fib_lookup(dev_net(ctx->rxq->dev), params,
                                           flags, true);
#endif
#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6:
                return bpf_ipv6_fib_lookup(dev_net(ctx->rxq->dev), params,
                                           flags, true);
#endif
        }
        return -EAFNOSUPPORT;
}

static const struct bpf_func_proto bpf_xdp_fib_lookup_proto = {
        .func                = bpf_xdp_fib_lookup,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_MEM,
        .arg3_type      = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
           struct bpf_fib_lookup *, params, int, plen, u32, flags)
{
        struct net *net = dev_net(skb->dev);
        int rc = -EAFNOSUPPORT;
        bool check_mtu = false;

        if (plen < sizeof(*params))
                return -EINVAL;

        if (flags & ~BPF_FIB_LOOKUP_MASK)
                return -EINVAL;

        if (params->tot_len)
                check_mtu = true;

        switch (params->family) {
#if IS_ENABLED(CONFIG_INET)
        case AF_INET:
                rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
                break;
#endif
#if IS_ENABLED(CONFIG_IPV6)
        case AF_INET6:
                rc = bpf_ipv6_fib_lookup(net, params, flags, check_mtu);
                break;
#endif
        }

        if (rc == BPF_FIB_LKUP_RET_SUCCESS && !check_mtu) {
                struct net_device *dev;

                /* When tot_len isn't provided by user, check skb
                 * against MTU of FIB lookup resulting net_device
                 */
                dev = dev_get_by_index_rcu(net, params->ifindex);
                if (!is_skb_forwardable(dev, skb))
                        rc = BPF_FIB_LKUP_RET_FRAG_NEEDED;

                params->mtu_result = dev->mtu; /* union with tot_len */
        }

        return rc;
}

static const struct bpf_func_proto bpf_skb_fib_lookup_proto = {
        .func                = bpf_skb_fib_lookup,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_MEM,
        .arg3_type      = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

static struct net_device *__dev_via_ifindex(struct net_device *dev_curr,
                                            u32 ifindex)
{
        struct net *netns = dev_net(dev_curr);

        /* Non-redirect use-cases can use ifindex=0 and save ifindex lookup */
        if (ifindex == 0)
                return dev_curr;

        return dev_get_by_index_rcu(netns, ifindex);
}

BPF_CALL_5(bpf_skb_check_mtu, struct sk_buff *, skb,
           u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
{
        int ret = BPF_MTU_CHK_RET_FRAG_NEEDED;
        struct net_device *dev = skb->dev;
        int mtu, dev_len, skb_len;

        if (unlikely(flags & ~(BPF_MTU_CHK_SEGS)))
                return -EINVAL;
        if (unlikely(flags & BPF_MTU_CHK_SEGS && (len_diff || *mtu_len)))
                return -EINVAL;

        dev = __dev_via_ifindex(dev, ifindex);
        if (unlikely(!dev))
                return -ENODEV;

        mtu = READ_ONCE(dev->mtu);
        dev_len = mtu + dev->hard_header_len;

        /* If set use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
        skb_len = *mtu_len ? *mtu_len + dev->hard_header_len : skb->len;

        skb_len += len_diff; /* minus result pass check */
        if (skb_len <= dev_len) {
                ret = BPF_MTU_CHK_RET_SUCCESS;
                goto out;
        }
        /* At this point, skb->len exceed MTU, but as it include length of all
         * segments, it can still be below MTU.  The SKB can possibly get
         * re-segmented in transmit path (see validate_xmit_skb).  Thus, user
         * must choose if segs are to be MTU checked.
         */
        if (skb_is_gso(skb)) {
                ret = BPF_MTU_CHK_RET_SUCCESS;
                if (flags & BPF_MTU_CHK_SEGS &&
                    !skb_gso_validate_network_len(skb, mtu))
                        ret = BPF_MTU_CHK_RET_SEGS_TOOBIG;
        }
out:
        *mtu_len = mtu;
        return ret;
}

BPF_CALL_5(bpf_xdp_check_mtu, struct xdp_buff *, xdp,
           u32, ifindex, u32 *, mtu_len, s32, len_diff, u64, flags)
{
        struct net_device *dev = xdp->rxq->dev;
        int xdp_len = xdp->data_end - xdp->data;
        int ret = BPF_MTU_CHK_RET_SUCCESS;
        int mtu, dev_len;

        /* XDP variant doesn't support multi-buffer segment check (yet) */
        if (unlikely(flags))
                return -EINVAL;

        dev = __dev_via_ifindex(dev, ifindex);
        if (unlikely(!dev))
                return -ENODEV;

        mtu = READ_ONCE(dev->mtu);
        dev_len = mtu + dev->hard_header_len;

        /* Use *mtu_len as input, L3 as iph->tot_len (like fib_lookup) */
        if (*mtu_len)
                xdp_len = *mtu_len + dev->hard_header_len;

        xdp_len += len_diff; /* minus result pass check */
        if (xdp_len > dev_len)
                ret = BPF_MTU_CHK_RET_FRAG_NEEDED;

        *mtu_len = mtu;
        return ret;
}

static const struct bpf_func_proto bpf_skb_check_mtu_proto = {
        .func                = bpf_skb_check_mtu,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
        .arg3_size        = sizeof(u32),
        .arg4_type      = ARG_ANYTHING,
        .arg5_type      = ARG_ANYTHING,
};

static const struct bpf_func_proto bpf_xdp_check_mtu_proto = {
        .func                = bpf_xdp_check_mtu,
        .gpl_only        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_PTR_TO_FIXED_SIZE_MEM | MEM_WRITE | MEM_ALIGNED,
        .arg3_size        = sizeof(u32),
        .arg4_type      = ARG_ANYTHING,
        .arg5_type      = ARG_ANYTHING,
};

#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
static int bpf_push_seg6_encap(struct sk_buff *skb, u32 type, void *hdr, u32 len)
{
        int err;
        struct ipv6_sr_hdr *srh = (struct ipv6_sr_hdr *)hdr;

        if (!seg6_validate_srh(srh, len, false))
                return -EINVAL;

        switch (type) {
        case BPF_LWT_ENCAP_SEG6_INLINE:
                if (skb->protocol != htons(ETH_P_IPV6))
                        return -EBADMSG;

                err = seg6_do_srh_inline(skb, srh);
                break;
        case BPF_LWT_ENCAP_SEG6:
                skb_reset_inner_headers(skb);
                skb->encapsulation = 1;
                err = seg6_do_srh_encap(skb, srh, IPPROTO_IPV6);
                break;
        default:
                return -EINVAL;
        }

        bpf_compute_data_pointers(skb);
        if (err)
                return err;

        skb_set_transport_header(skb, sizeof(struct ipv6hdr));

        return seg6_lookup_nexthop(skb, NULL, 0);
}
#endif /* CONFIG_IPV6_SEG6_BPF */

#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
static int bpf_push_ip_encap(struct sk_buff *skb, void *hdr, u32 len,
                             bool ingress)
{
        return bpf_lwt_push_ip_encap(skb, hdr, len, ingress);
}
#endif

BPF_CALL_4(bpf_lwt_in_push_encap, struct sk_buff *, skb, u32, type, void *, hdr,
           u32, len)
{
        switch (type) {
#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
        case BPF_LWT_ENCAP_SEG6:
        case BPF_LWT_ENCAP_SEG6_INLINE:
                return bpf_push_seg6_encap(skb, type, hdr, len);
#endif
#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
        case BPF_LWT_ENCAP_IP:
                return bpf_push_ip_encap(skb, hdr, len, true /* ingress */);
#endif
        default:
                return -EINVAL;
        }
}

BPF_CALL_4(bpf_lwt_xmit_push_encap, struct sk_buff *, skb, u32, type,
           void *, hdr, u32, len)
{
        switch (type) {
#if IS_ENABLED(CONFIG_LWTUNNEL_BPF)
        case BPF_LWT_ENCAP_IP:
                return bpf_push_ip_encap(skb, hdr, len, false /* egress */);
#endif
        default:
                return -EINVAL;
        }
}

static const struct bpf_func_proto bpf_lwt_in_push_encap_proto = {
        .func                = bpf_lwt_in_push_encap,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE
};

static const struct bpf_func_proto bpf_lwt_xmit_push_encap_proto = {
        .func                = bpf_lwt_xmit_push_encap,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE
};

#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
BPF_CALL_4(bpf_lwt_seg6_store_bytes, struct sk_buff *, skb, u32, offset,
           const void *, from, u32, len)
{
        struct seg6_bpf_srh_state *srh_state =
                this_cpu_ptr(&seg6_bpf_srh_states);
        struct ipv6_sr_hdr *srh = srh_state->srh;
        void *srh_tlvs, *srh_end, *ptr;
        int srhoff = 0;

        lockdep_assert_held(&srh_state->bh_lock);
        if (srh == NULL)
                return -EINVAL;

        srh_tlvs = (void *)((char *)srh + ((srh->first_segment + 1) << 4));
        srh_end = (void *)((char *)srh + sizeof(*srh) + srh_state->hdrlen);

        ptr = skb->data + offset;
        if (ptr >= srh_tlvs && ptr + len <= srh_end)
                srh_state->valid = false;
        else if (ptr < (void *)&srh->flags ||
                 ptr + len > (void *)&srh->segments)
                return -EFAULT;

        if (unlikely(bpf_try_make_writable(skb, offset + len)))
                return -EFAULT;
        if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
                return -EINVAL;
        srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);

        memcpy(skb->data + offset, from, len);
        return 0;
}

static const struct bpf_func_proto bpf_lwt_seg6_store_bytes_proto = {
        .func                = bpf_lwt_seg6_store_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE
};

static void bpf_update_srh_state(struct sk_buff *skb)
{
        struct seg6_bpf_srh_state *srh_state =
                this_cpu_ptr(&seg6_bpf_srh_states);
        int srhoff = 0;

        if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0) {
                srh_state->srh = NULL;
        } else {
                srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
                srh_state->hdrlen = srh_state->srh->hdrlen << 3;
                srh_state->valid = true;
        }
}

BPF_CALL_4(bpf_lwt_seg6_action, struct sk_buff *, skb,
           u32, action, void *, param, u32, param_len)
{
        struct seg6_bpf_srh_state *srh_state =
                this_cpu_ptr(&seg6_bpf_srh_states);
        int hdroff = 0;
        int err;

        lockdep_assert_held(&srh_state->bh_lock);
        switch (action) {
        case SEG6_LOCAL_ACTION_END_X:
                if (!seg6_bpf_has_valid_srh(skb))
                        return -EBADMSG;
                if (param_len != sizeof(struct in6_addr))
                        return -EINVAL;
                return seg6_lookup_nexthop(skb, (struct in6_addr *)param, 0);
        case SEG6_LOCAL_ACTION_END_T:
                if (!seg6_bpf_has_valid_srh(skb))
                        return -EBADMSG;
                if (param_len != sizeof(int))
                        return -EINVAL;
                return seg6_lookup_nexthop(skb, NULL, *(int *)param);
        case SEG6_LOCAL_ACTION_END_DT6:
                if (!seg6_bpf_has_valid_srh(skb))
                        return -EBADMSG;
                if (param_len != sizeof(int))
                        return -EINVAL;

                if (ipv6_find_hdr(skb, &hdroff, IPPROTO_IPV6, NULL, NULL) < 0)
                        return -EBADMSG;
                if (!pskb_pull(skb, hdroff))
                        return -EBADMSG;

                skb_postpull_rcsum(skb, skb_network_header(skb), hdroff);
                skb_reset_network_header(skb);
                skb_reset_transport_header(skb);
                skb->encapsulation = 0;

                bpf_compute_data_pointers(skb);
                bpf_update_srh_state(skb);
                return seg6_lookup_nexthop(skb, NULL, *(int *)param);
        case SEG6_LOCAL_ACTION_END_B6:
                if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
                        return -EBADMSG;
                err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6_INLINE,
                                          param, param_len);
                if (!err)
                        bpf_update_srh_state(skb);

                return err;
        case SEG6_LOCAL_ACTION_END_B6_ENCAP:
                if (srh_state->srh && !seg6_bpf_has_valid_srh(skb))
                        return -EBADMSG;
                err = bpf_push_seg6_encap(skb, BPF_LWT_ENCAP_SEG6,
                                          param, param_len);
                if (!err)
                        bpf_update_srh_state(skb);

                return err;
        default:
                return -EINVAL;
        }
}

static const struct bpf_func_proto bpf_lwt_seg6_action_proto = {
        .func                = bpf_lwt_seg6_action,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg4_type        = ARG_CONST_SIZE
};

BPF_CALL_3(bpf_lwt_seg6_adjust_srh, struct sk_buff *, skb, u32, offset,
           s32, len)
{
        struct seg6_bpf_srh_state *srh_state =
                this_cpu_ptr(&seg6_bpf_srh_states);
        struct ipv6_sr_hdr *srh = srh_state->srh;
        void *srh_end, *srh_tlvs, *ptr;
        struct ipv6hdr *hdr;
        int srhoff = 0;
        int ret;

        lockdep_assert_held(&srh_state->bh_lock);
        if (unlikely(srh == NULL))
                return -EINVAL;

        srh_tlvs = (void *)((unsigned char *)srh + sizeof(*srh) +
                        ((srh->first_segment + 1) << 4));
        srh_end = (void *)((unsigned char *)srh + sizeof(*srh) +
                        srh_state->hdrlen);
        ptr = skb->data + offset;

        if (unlikely(ptr < srh_tlvs || ptr > srh_end))
                return -EFAULT;
        if (unlikely(len < 0 && (void *)((char *)ptr - len) > srh_end))
                return -EFAULT;

        if (len > 0) {
                ret = skb_cow_head(skb, len);
                if (unlikely(ret < 0))
                        return ret;

                ret = bpf_skb_net_hdr_push(skb, offset, len);
        } else {
                ret = bpf_skb_net_hdr_pop(skb, offset, -1 * len);
        }

        bpf_compute_data_pointers(skb);
        if (unlikely(ret < 0))
                return ret;

        hdr = (struct ipv6hdr *)skb->data;
        hdr->payload_len = htons(skb->len - sizeof(struct ipv6hdr));

        if (ipv6_find_hdr(skb, &srhoff, IPPROTO_ROUTING, NULL, NULL) < 0)
                return -EINVAL;
        srh_state->srh = (struct ipv6_sr_hdr *)(skb->data + srhoff);
        srh_state->hdrlen += len;
        srh_state->valid = false;
        return 0;
}

static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
        .func                = bpf_lwt_seg6_adjust_srh,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};
#endif /* CONFIG_IPV6_SEG6_BPF */

#ifdef CONFIG_INET
static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
                              int dif, int sdif, u8 family, u8 proto)
{
        struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
        bool refcounted = false;
        struct sock *sk = NULL;

        if (family == AF_INET) {
                __be32 src4 = tuple->ipv4.saddr;
                __be32 dst4 = tuple->ipv4.daddr;

                if (proto == IPPROTO_TCP)
                        sk = __inet_lookup(net, hinfo, NULL, 0,
                                           src4, tuple->ipv4.sport,
                                           dst4, tuple->ipv4.dport,
                                           dif, sdif, &refcounted);
                else
                        sk = __udp4_lib_lookup(net, src4, tuple->ipv4.sport,
                                               dst4, tuple->ipv4.dport,
                                               dif, sdif, net->ipv4.udp_table, NULL);
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                struct in6_addr *src6 = (struct in6_addr *)&tuple->ipv6.saddr;
                struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;

                if (proto == IPPROTO_TCP)
                        sk = __inet6_lookup(net, hinfo, NULL, 0,
                                            src6, tuple->ipv6.sport,
                                            dst6, ntohs(tuple->ipv6.dport),
                                            dif, sdif, &refcounted);
                else if (likely(ipv6_bpf_stub))
                        sk = ipv6_bpf_stub->udp6_lib_lookup(net,
                                                            src6, tuple->ipv6.sport,
                                                            dst6, tuple->ipv6.dport,
                                                            dif, sdif,
                                                            net->ipv4.udp_table, NULL);
#endif
        }

        if (unlikely(sk && !refcounted && !sock_flag(sk, SOCK_RCU_FREE))) {
                WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
                sk = NULL;
        }
        return sk;
}

/* bpf_skc_lookup performs the core lookup for different types of sockets,
 * taking a reference on the socket if it doesn't have the flag SOCK_RCU_FREE.
 */
static struct sock *
__bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
                 struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
                 u64 flags, int sdif)
{
        struct sock *sk = NULL;
        struct net *net;
        u8 family;

        if (len == sizeof(tuple->ipv4))
                family = AF_INET;
        else if (len == sizeof(tuple->ipv6))
                family = AF_INET6;
        else
                return NULL;

        if (unlikely(flags || !((s32)netns_id < 0 || netns_id <= S32_MAX)))
                goto out;

        if (sdif < 0) {
                if (family == AF_INET)
                        sdif = inet_sdif(skb);
                else
                        sdif = inet6_sdif(skb);
        }

        if ((s32)netns_id < 0) {
                net = caller_net;
                sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
        } else {
                net = get_net_ns_by_id(caller_net, netns_id);
                if (unlikely(!net))
                        goto out;
                sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);
                put_net(net);
        }

out:
        return sk;
}

static struct sock *
__bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
                struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,
                u64 flags, int sdif)
{
        struct sock *sk = __bpf_skc_lookup(skb, tuple, len, caller_net,
                                           ifindex, proto, netns_id, flags,
                                           sdif);

        if (sk) {
                struct sock *sk2 = sk_to_full_sk(sk);

                /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk
                 * sock refcnt is decremented to prevent a request_sock leak.
                 */
                if (sk2 != sk) {
                        sock_gen_put(sk);
                        /* Ensure there is no need to bump sk2 refcnt */
                        if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) {
                                WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
                                return NULL;
                        }
                        sk = sk2;
                }
        }

        return sk;
}

static struct sock *
bpf_skc_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
               u8 proto, u64 netns_id, u64 flags)
{
        struct net *caller_net;
        int ifindex;

        if (skb->dev) {
                caller_net = dev_net(skb->dev);
                ifindex = skb->dev->ifindex;
        } else {
                caller_net = sock_net(skb->sk);
                ifindex = 0;
        }

        return __bpf_skc_lookup(skb, tuple, len, caller_net, ifindex, proto,
                                netns_id, flags, -1);
}

static struct sock *
bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,
              u8 proto, u64 netns_id, u64 flags)
{
        struct sock *sk = bpf_skc_lookup(skb, tuple, len, proto, netns_id,
                                         flags);

        if (sk) {
                struct sock *sk2 = sk_to_full_sk(sk);

                /* sk_to_full_sk() may return (sk)->rsk_listener, so make sure the original sk
                 * sock refcnt is decremented to prevent a request_sock leak.
                 */
                if (sk2 != sk) {
                        sock_gen_put(sk);
                        /* Ensure there is no need to bump sk2 refcnt */
                        if (unlikely(sk2 && !sock_flag(sk2, SOCK_RCU_FREE))) {
                                WARN_ONCE(1, "Found non-RCU, unreferenced socket!");
                                return NULL;
                        }
                        sk = sk2;
                }
        }

        return sk;
}

BPF_CALL_5(bpf_skc_lookup_tcp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)bpf_skc_lookup(skb, tuple, len, IPPROTO_TCP,
                                             netns_id, flags);
}

static const struct bpf_func_proto bpf_skc_lookup_tcp_proto = {
        .func                = bpf_skc_lookup_tcp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCK_COMMON_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_sk_lookup_tcp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_TCP,
                                            netns_id, flags);
}

static const struct bpf_func_proto bpf_sk_lookup_tcp_proto = {
        .func                = bpf_sk_lookup_tcp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_sk_lookup_udp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)bpf_sk_lookup(skb, tuple, len, IPPROTO_UDP,
                                            netns_id, flags);
}

static const struct bpf_func_proto bpf_sk_lookup_udp_proto = {
        .func                = bpf_sk_lookup_udp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_tc_skc_lookup_tcp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        struct net_device *dev = skb->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_skc_lookup(skb, tuple, len, caller_net,
                                               ifindex, IPPROTO_TCP, netns_id,
                                               flags, sdif);
}

static const struct bpf_func_proto bpf_tc_skc_lookup_tcp_proto = {
        .func                = bpf_tc_skc_lookup_tcp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCK_COMMON_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_tc_sk_lookup_tcp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        struct net_device *dev = skb->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
                                              ifindex, IPPROTO_TCP, netns_id,
                                              flags, sdif);
}

static const struct bpf_func_proto bpf_tc_sk_lookup_tcp_proto = {
        .func                = bpf_tc_sk_lookup_tcp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_tc_sk_lookup_udp, struct sk_buff *, skb,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        struct net_device *dev = skb->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_sk_lookup(skb, tuple, len, caller_net,
                                              ifindex, IPPROTO_UDP, netns_id,
                                              flags, sdif);
}

static const struct bpf_func_proto bpf_tc_sk_lookup_udp_proto = {
        .func                = bpf_tc_sk_lookup_udp,
        .gpl_only        = false,
        .pkt_access        = true,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_1(bpf_sk_release, struct sock *, sk)
{
        if (sk && sk_is_refcounted(sk))
                sock_gen_put(sk);
        return 0;
}

static const struct bpf_func_proto bpf_sk_release_proto = {
        .func                = bpf_sk_release,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON | OBJ_RELEASE,
};

BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
{
        struct net_device *dev = ctx->rxq->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
                                              ifindex, IPPROTO_UDP, netns_id,
                                              flags, sdif);
}

static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {
        .func           = bpf_xdp_sk_lookup_udp,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type      = ARG_ANYTHING,
        .arg5_type      = ARG_ANYTHING,
};

BPF_CALL_5(bpf_xdp_skc_lookup_tcp, struct xdp_buff *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
{
        struct net_device *dev = ctx->rxq->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_skc_lookup(NULL, tuple, len, caller_net,
                                               ifindex, IPPROTO_TCP, netns_id,
                                               flags, sdif);
}

static const struct bpf_func_proto bpf_xdp_skc_lookup_tcp_proto = {
        .func           = bpf_xdp_skc_lookup_tcp,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_PTR_TO_SOCK_COMMON_OR_NULL,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type      = ARG_ANYTHING,
        .arg5_type      = ARG_ANYTHING,
};

BPF_CALL_5(bpf_xdp_sk_lookup_tcp, struct xdp_buff *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)
{
        struct net_device *dev = ctx->rxq->dev;
        int ifindex = dev->ifindex, sdif = dev_sdif(dev);
        struct net *caller_net = dev_net(dev);

        return (unsigned long)__bpf_sk_lookup(NULL, tuple, len, caller_net,
                                              ifindex, IPPROTO_TCP, netns_id,
                                              flags, sdif);
}

static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {
        .func           = bpf_xdp_sk_lookup_tcp,
        .gpl_only       = false,
        .pkt_access     = true,
        .ret_type       = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type      = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type      = ARG_ANYTHING,
        .arg5_type      = ARG_ANYTHING,
};

BPF_CALL_5(bpf_sock_addr_skc_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)__bpf_skc_lookup(NULL, tuple, len,
                                               sock_net(ctx->sk), 0,
                                               IPPROTO_TCP, netns_id, flags,
                                               -1);
}

static const struct bpf_func_proto bpf_sock_addr_skc_lookup_tcp_proto = {
        .func                = bpf_sock_addr_skc_lookup_tcp,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_SOCK_COMMON_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_sock_addr_sk_lookup_tcp, struct bpf_sock_addr_kern *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
                                              sock_net(ctx->sk), 0, IPPROTO_TCP,
                                              netns_id, flags, -1);
}

static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {
        .func                = bpf_sock_addr_sk_lookup_tcp,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

BPF_CALL_5(bpf_sock_addr_sk_lookup_udp, struct bpf_sock_addr_kern *, ctx,
           struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)
{
        return (unsigned long)__bpf_sk_lookup(NULL, tuple, len,
                                              sock_net(ctx->sk), 0, IPPROTO_UDP,
                                              netns_id, flags, -1);
}

static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {
        .func                = bpf_sock_addr_sk_lookup_udp,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
        .arg4_type        = ARG_ANYTHING,
        .arg5_type        = ARG_ANYTHING,
};

bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                                  struct bpf_insn_access_aux *info)
{
        if (off < 0 || off >= offsetofend(struct bpf_tcp_sock,
                                          icsk_retransmits))
                return false;

        if (off % size != 0)
                return false;

        switch (off) {
        case offsetof(struct bpf_tcp_sock, bytes_received):
        case offsetof(struct bpf_tcp_sock, bytes_acked):
                return size == sizeof(__u64);
        default:
                return size == sizeof(__u32);
        }
}

u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type,
                                    const struct bpf_insn *si,
                                    struct bpf_insn *insn_buf,
                                    struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

#define BPF_TCP_SOCK_GET_COMMON(FIELD)                                        \
        do {                                                                \
                BUILD_BUG_ON(sizeof_field(struct tcp_sock, FIELD) >        \
                             sizeof_field(struct bpf_tcp_sock, FIELD));        \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_sock, FIELD),\
                                      si->dst_reg, si->src_reg,                \
                                      offsetof(struct tcp_sock, FIELD)); \
        } while (0)

#define BPF_INET_SOCK_GET_COMMON(FIELD)                                        \
        do {                                                                \
                BUILD_BUG_ON(sizeof_field(struct inet_connection_sock,        \
                                          FIELD) >                        \
                             sizeof_field(struct bpf_tcp_sock, FIELD));        \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                        \
                                        struct inet_connection_sock,        \
                                        FIELD),                                \
                                      si->dst_reg, si->src_reg,                \
                                      offsetof(                                \
                                        struct inet_connection_sock,        \
                                        FIELD));                        \
        } while (0)

        BTF_TYPE_EMIT(struct bpf_tcp_sock);

        switch (si->off) {
        case offsetof(struct bpf_tcp_sock, rtt_min):
                BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
                             sizeof(struct minmax));
                BUILD_BUG_ON(sizeof(struct minmax) <
                             sizeof(struct minmax_sample));

                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      offsetof(struct tcp_sock, rtt_min) +
                                      offsetof(struct minmax_sample, v));
                break;
        case offsetof(struct bpf_tcp_sock, snd_cwnd):
                BPF_TCP_SOCK_GET_COMMON(snd_cwnd);
                break;
        case offsetof(struct bpf_tcp_sock, srtt_us):
                BPF_TCP_SOCK_GET_COMMON(srtt_us);
                break;
        case offsetof(struct bpf_tcp_sock, snd_ssthresh):
                BPF_TCP_SOCK_GET_COMMON(snd_ssthresh);
                break;
        case offsetof(struct bpf_tcp_sock, rcv_nxt):
                BPF_TCP_SOCK_GET_COMMON(rcv_nxt);
                break;
        case offsetof(struct bpf_tcp_sock, snd_nxt):
                BPF_TCP_SOCK_GET_COMMON(snd_nxt);
                break;
        case offsetof(struct bpf_tcp_sock, snd_una):
                BPF_TCP_SOCK_GET_COMMON(snd_una);
                break;
        case offsetof(struct bpf_tcp_sock, mss_cache):
                BPF_TCP_SOCK_GET_COMMON(mss_cache);
                break;
        case offsetof(struct bpf_tcp_sock, ecn_flags):
                BPF_TCP_SOCK_GET_COMMON(ecn_flags);
                break;
        case offsetof(struct bpf_tcp_sock, rate_delivered):
                BPF_TCP_SOCK_GET_COMMON(rate_delivered);
                break;
        case offsetof(struct bpf_tcp_sock, rate_interval_us):
                BPF_TCP_SOCK_GET_COMMON(rate_interval_us);
                break;
        case offsetof(struct bpf_tcp_sock, packets_out):
                BPF_TCP_SOCK_GET_COMMON(packets_out);
                break;
        case offsetof(struct bpf_tcp_sock, retrans_out):
                BPF_TCP_SOCK_GET_COMMON(retrans_out);
                break;
        case offsetof(struct bpf_tcp_sock, total_retrans):
                BPF_TCP_SOCK_GET_COMMON(total_retrans);
                break;
        case offsetof(struct bpf_tcp_sock, segs_in):
                BPF_TCP_SOCK_GET_COMMON(segs_in);
                break;
        case offsetof(struct bpf_tcp_sock, data_segs_in):
                BPF_TCP_SOCK_GET_COMMON(data_segs_in);
                break;
        case offsetof(struct bpf_tcp_sock, segs_out):
                BPF_TCP_SOCK_GET_COMMON(segs_out);
                break;
        case offsetof(struct bpf_tcp_sock, data_segs_out):
                BPF_TCP_SOCK_GET_COMMON(data_segs_out);
                break;
        case offsetof(struct bpf_tcp_sock, lost_out):
                BPF_TCP_SOCK_GET_COMMON(lost_out);
                break;
        case offsetof(struct bpf_tcp_sock, sacked_out):
                BPF_TCP_SOCK_GET_COMMON(sacked_out);
                break;
        case offsetof(struct bpf_tcp_sock, bytes_received):
                BPF_TCP_SOCK_GET_COMMON(bytes_received);
                break;
        case offsetof(struct bpf_tcp_sock, bytes_acked):
                BPF_TCP_SOCK_GET_COMMON(bytes_acked);
                break;
        case offsetof(struct bpf_tcp_sock, dsack_dups):
                BPF_TCP_SOCK_GET_COMMON(dsack_dups);
                break;
        case offsetof(struct bpf_tcp_sock, delivered):
                BPF_TCP_SOCK_GET_COMMON(delivered);
                break;
        case offsetof(struct bpf_tcp_sock, delivered_ce):
                BPF_TCP_SOCK_GET_COMMON(delivered_ce);
                break;
        case offsetof(struct bpf_tcp_sock, icsk_retransmits):
                BPF_INET_SOCK_GET_COMMON(icsk_retransmits);
                break;
        }

        return insn - insn_buf;
}

BPF_CALL_1(bpf_tcp_sock, struct sock *, sk)
{
        if (sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_tcp_sock_proto = {
        .func                = bpf_tcp_sock,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_TCP_SOCK_OR_NULL,
        .arg1_type        = ARG_PTR_TO_SOCK_COMMON,
};

BPF_CALL_1(bpf_get_listener_sock, struct sock *, sk)
{
        sk = sk_to_full_sk(sk);

        if (sk && sk->sk_state == TCP_LISTEN && sock_flag(sk, SOCK_RCU_FREE))
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

static const struct bpf_func_proto bpf_get_listener_sock_proto = {
        .func                = bpf_get_listener_sock,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_SOCKET_OR_NULL,
        .arg1_type        = ARG_PTR_TO_SOCK_COMMON,
};

BPF_CALL_1(bpf_skb_ecn_set_ce, struct sk_buff *, skb)
{
        unsigned int iphdr_len;

        switch (skb_protocol(skb, true)) {
        case cpu_to_be16(ETH_P_IP):
                iphdr_len = sizeof(struct iphdr);
                break;
        case cpu_to_be16(ETH_P_IPV6):
                iphdr_len = sizeof(struct ipv6hdr);
                break;
        default:
                return 0;
        }

        if (skb_headlen(skb) < iphdr_len)
                return 0;

        if (skb_cloned(skb) && !skb_clone_writable(skb, iphdr_len))
                return 0;

        return INET_ECN_set_ce(skb);
}

bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                                  struct bpf_insn_access_aux *info)
{
        if (off < 0 || off >= offsetofend(struct bpf_xdp_sock, queue_id))
                return false;

        if (off % size != 0)
                return false;

        switch (off) {
        default:
                return size == sizeof(__u32);
        }
}

u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
                                    const struct bpf_insn *si,
                                    struct bpf_insn *insn_buf,
                                    struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

#define BPF_XDP_SOCK_GET(FIELD)                                                \
        do {                                                                \
                BUILD_BUG_ON(sizeof_field(struct xdp_sock, FIELD) >        \
                             sizeof_field(struct bpf_xdp_sock, FIELD));        \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_sock, FIELD),\
                                      si->dst_reg, si->src_reg,                \
                                      offsetof(struct xdp_sock, FIELD)); \
        } while (0)

        switch (si->off) {
        case offsetof(struct bpf_xdp_sock, queue_id):
                BPF_XDP_SOCK_GET(queue_id);
                break;
        }

        return insn - insn_buf;
}

static const struct bpf_func_proto bpf_skb_ecn_set_ce_proto = {
        .func           = bpf_skb_ecn_set_ce,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
};

BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
           struct tcphdr *, th, u32, th_len)
{
#ifdef CONFIG_SYN_COOKIES
        int ret;

        if (unlikely(!sk || th_len < sizeof(*th)))
                return -EINVAL;

        /* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */
        if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
                return -EINVAL;

        if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
                return -EINVAL;

        if (!th->ack || th->rst || th->syn)
                return -ENOENT;

        if (unlikely(iph_len < sizeof(struct iphdr)))
                return -EINVAL;

        if (tcp_synq_no_recent_overflow(sk))
                return -ENOENT;

        /* Both struct iphdr and struct ipv6hdr have the version field at the
         * same offset so we can cast to the shorter header (struct iphdr).
         */
        switch (((struct iphdr *)iph)->version) {
        case 4:
                if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
                        return -EINVAL;

                ret = __cookie_v4_check((struct iphdr *)iph, th);
                break;

#if IS_BUILTIN(CONFIG_IPV6)
        case 6:
                if (unlikely(iph_len < sizeof(struct ipv6hdr)))
                        return -EINVAL;

                if (sk->sk_family != AF_INET6)
                        return -EINVAL;

                ret = __cookie_v6_check((struct ipv6hdr *)iph, th);
                break;
#endif /* CONFIG_IPV6 */

        default:
                return -EPROTONOSUPPORT;
        }

        if (ret > 0)
                return 0;

        return -ENOENT;
#else
        return -ENOTSUPP;
#endif
}

static const struct bpf_func_proto bpf_tcp_check_syncookie_proto = {
        .func                = bpf_tcp_check_syncookie,
        .gpl_only        = true,
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len,
           struct tcphdr *, th, u32, th_len)
{
#ifdef CONFIG_SYN_COOKIES
        u32 cookie;
        u16 mss;

        if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4))
                return -EINVAL;

        if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
                return -EINVAL;

        if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies))
                return -ENOENT;

        if (!th->syn || th->ack || th->fin || th->rst)
                return -EINVAL;

        if (unlikely(iph_len < sizeof(struct iphdr)))
                return -EINVAL;

        /* Both struct iphdr and struct ipv6hdr have the version field at the
         * same offset so we can cast to the shorter header (struct iphdr).
         */
        switch (((struct iphdr *)iph)->version) {
        case 4:
                if (sk->sk_family == AF_INET6 && ipv6_only_sock(sk))
                        return -EINVAL;

                mss = tcp_v4_get_syncookie(sk, iph, th, &cookie);
                break;

#if IS_BUILTIN(CONFIG_IPV6)
        case 6:
                if (unlikely(iph_len < sizeof(struct ipv6hdr)))
                        return -EINVAL;

                if (sk->sk_family != AF_INET6)
                        return -EINVAL;

                mss = tcp_v6_get_syncookie(sk, iph, th, &cookie);
                break;
#endif /* CONFIG_IPV6 */

        default:
                return -EPROTONOSUPPORT;
        }
        if (mss == 0)
                return -ENOENT;

        return cookie | ((u64)mss << 32);
#else
        return -EOPNOTSUPP;
#endif /* CONFIG_SYN_COOKIES */
}

static const struct bpf_func_proto bpf_tcp_gen_syncookie_proto = {
        .func                = bpf_tcp_gen_syncookie,
        .gpl_only        = true, /* __cookie_v*_init_sequence() is GPL */
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg5_type        = ARG_CONST_SIZE,
};

BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags)
{
        if (!sk || flags != 0)
                return -EINVAL;
        if (!skb_at_tc_ingress(skb))
                return -EOPNOTSUPP;
        if (unlikely(dev_net(skb->dev) != sock_net(sk)))
                return -ENETUNREACH;
        if (sk_unhashed(sk))
                return -EOPNOTSUPP;
        if (sk_is_refcounted(sk) &&
            unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
                return -ENOENT;

        skb_orphan(skb);
        skb->sk = sk;
        skb->destructor = sock_pfree;

        return 0;
}

static const struct bpf_func_proto bpf_sk_assign_proto = {
        .func                = bpf_sk_assign,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .arg3_type        = ARG_ANYTHING,
};

static const u8 *bpf_search_tcp_opt(const u8 *op, const u8 *opend,
                                    u8 search_kind, const u8 *magic,
                                    u8 magic_len, bool *eol)
{
        u8 kind, kind_len;

        *eol = false;

        while (op < opend) {
                kind = op[0];

                if (kind == TCPOPT_EOL) {
                        *eol = true;
                        return ERR_PTR(-ENOMSG);
                } else if (kind == TCPOPT_NOP) {
                        op++;
                        continue;
                }

                if (opend - op < 2 || opend - op < op[1] || op[1] < 2)
                        /* Something is wrong in the received header.
                         * Follow the TCP stack's tcp_parse_options()
                         * and just bail here.
                         */
                        return ERR_PTR(-EFAULT);

                kind_len = op[1];
                if (search_kind == kind) {
                        if (!magic_len)
                                return op;

                        if (magic_len > kind_len - 2)
                                return ERR_PTR(-ENOMSG);

                        if (!memcmp(&op[2], magic, magic_len))
                                return op;
                }

                op += kind_len;
        }

        return ERR_PTR(-ENOMSG);
}

BPF_CALL_4(bpf_sock_ops_load_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
           void *, search_res, u32, len, u64, flags)
{
        bool eol, load_syn = flags & BPF_LOAD_HDR_OPT_TCP_SYN;
        const u8 *op, *opend, *magic, *search = search_res;
        u8 search_kind, search_len, copy_len, magic_len;
        int ret;

        if (!is_locked_tcp_sock_ops(bpf_sock))
                return -EOPNOTSUPP;

        /* 2 byte is the minimal option len except TCPOPT_NOP and
         * TCPOPT_EOL which are useless for the bpf prog to learn
         * and this helper disallow loading them also.
         */
        if (len < 2 || flags & ~BPF_LOAD_HDR_OPT_TCP_SYN)
                return -EINVAL;

        search_kind = search[0];
        search_len = search[1];

        if (search_len > len || search_kind == TCPOPT_NOP ||
            search_kind == TCPOPT_EOL)
                return -EINVAL;

        if (search_kind == TCPOPT_EXP || search_kind == 253) {
                /* 16 or 32 bit magic.  +2 for kind and kind length */
                if (search_len != 4 && search_len != 6)
                        return -EINVAL;
                magic = &search[2];
                magic_len = search_len - 2;
        } else {
                if (search_len)
                        return -EINVAL;
                magic = NULL;
                magic_len = 0;
        }

        if (load_syn) {
                ret = bpf_sock_ops_get_syn(bpf_sock, TCP_BPF_SYN, &op);
                if (ret < 0)
                        return ret;

                opend = op + ret;
                op += sizeof(struct tcphdr);
        } else {
                if (!bpf_sock->skb ||
                    bpf_sock->op == BPF_SOCK_OPS_HDR_OPT_LEN_CB)
                        /* This bpf_sock->op cannot call this helper */
                        return -EPERM;

                opend = bpf_sock->skb_data_end;
                op = bpf_sock->skb->data + sizeof(struct tcphdr);
        }

        op = bpf_search_tcp_opt(op, opend, search_kind, magic, magic_len,
                                &eol);
        if (IS_ERR(op))
                return PTR_ERR(op);

        copy_len = op[1];
        ret = copy_len;
        if (copy_len > len) {
                ret = -ENOSPC;
                copy_len = len;
        }

        memcpy(search_res, op, copy_len);
        return ret;
}

static const struct bpf_func_proto bpf_sock_ops_load_hdr_opt_proto = {
        .func                = bpf_sock_ops_load_hdr_opt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_WRITE,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_sock_ops_store_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
           const void *, from, u32, len, u64, flags)
{
        u8 new_kind, new_kind_len, magic_len = 0, *opend;
        const u8 *op, *new_op, *magic = NULL;
        struct sk_buff *skb;
        bool eol;

        if (bpf_sock->op != BPF_SOCK_OPS_WRITE_HDR_OPT_CB)
                return -EPERM;

        if (len < 2 || flags)
                return -EINVAL;

        new_op = from;
        new_kind = new_op[0];
        new_kind_len = new_op[1];

        if (new_kind_len > len || new_kind == TCPOPT_NOP ||
            new_kind == TCPOPT_EOL)
                return -EINVAL;

        if (new_kind_len > bpf_sock->remaining_opt_len)
                return -ENOSPC;

        /* 253 is another experimental kind */
        if (new_kind == TCPOPT_EXP || new_kind == 253)  {
                if (new_kind_len < 4)
                        return -EINVAL;
                /* Match for the 2 byte magic also.
                 * RFC 6994: the magic could be 2 or 4 bytes.
                 * Hence, matching by 2 byte only is on the
                 * conservative side but it is the right
                 * thing to do for the 'search-for-duplication'
                 * purpose.
                 */
                magic = &new_op[2];
                magic_len = 2;
        }

        /* Check for duplication */
        skb = bpf_sock->skb;
        op = skb->data + sizeof(struct tcphdr);
        opend = bpf_sock->skb_data_end;

        op = bpf_search_tcp_opt(op, opend, new_kind, magic, magic_len,
                                &eol);
        if (!IS_ERR(op))
                return -EEXIST;

        if (PTR_ERR(op) != -ENOMSG)
                return PTR_ERR(op);

        if (eol)
                /* The option has been ended.  Treat it as no more
                 * header option can be written.
                 */
                return -ENOSPC;

        /* No duplication found.  Store the header option. */
        memcpy(opend, from, new_kind_len);

        bpf_sock->remaining_opt_len -= new_kind_len;
        bpf_sock->skb_data_end += new_kind_len;

        return 0;
}

static const struct bpf_func_proto bpf_sock_ops_store_hdr_opt_proto = {
        .func                = bpf_sock_ops_store_hdr_opt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_MEM | MEM_RDONLY,
        .arg3_type        = ARG_CONST_SIZE,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_3(bpf_sock_ops_reserve_hdr_opt, struct bpf_sock_ops_kern *, bpf_sock,
           u32, len, u64, flags)
{
        if (bpf_sock->op != BPF_SOCK_OPS_HDR_OPT_LEN_CB)
                return -EPERM;

        if (flags || len < 2)
                return -EINVAL;

        if (len > bpf_sock->remaining_opt_len)
                return -ENOSPC;

        bpf_sock->remaining_opt_len -= len;

        return 0;
}

static const struct bpf_func_proto bpf_sock_ops_reserve_hdr_opt_proto = {
        .func                = bpf_sock_ops_reserve_hdr_opt,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_ANYTHING,
};

BPF_CALL_3(bpf_skb_set_tstamp, struct sk_buff *, skb,
           u64, tstamp, u32, tstamp_type)
{
        /* skb_clear_delivery_time() is done for inet protocol */
        if (skb->protocol != htons(ETH_P_IP) &&
            skb->protocol != htons(ETH_P_IPV6))
                return -EOPNOTSUPP;

        switch (tstamp_type) {
        case BPF_SKB_CLOCK_REALTIME:
                skb->tstamp = tstamp;
                skb->tstamp_type = SKB_CLOCK_REALTIME;
                break;
        case BPF_SKB_CLOCK_MONOTONIC:
                if (!tstamp)
                        return -EINVAL;
                skb->tstamp = tstamp;
                skb->tstamp_type = SKB_CLOCK_MONOTONIC;
                break;
        case BPF_SKB_CLOCK_TAI:
                if (!tstamp)
                        return -EINVAL;
                skb->tstamp = tstamp;
                skb->tstamp_type = SKB_CLOCK_TAI;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

static const struct bpf_func_proto bpf_skb_set_tstamp_proto = {
        .func           = bpf_skb_set_tstamp,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type      = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_ANYTHING,
        .arg3_type      = ARG_ANYTHING,
};

#ifdef CONFIG_SYN_COOKIES
BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv4, struct iphdr *, iph,
           struct tcphdr *, th, u32, th_len)
{
        u32 cookie;
        u16 mss;

        if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
                return -EINVAL;

        mss = tcp_parse_mss_option(th, 0) ?: TCP_MSS_DEFAULT;
        cookie = __cookie_v4_init_sequence(iph, th, &mss);

        return cookie | ((u64)mss << 32);
}

static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = {
        .func                = bpf_tcp_raw_gen_syncookie_ipv4,
        .gpl_only        = true, /* __cookie_v4_init_sequence() is GPL */
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg1_size        = sizeof(struct iphdr),
        .arg2_type        = ARG_PTR_TO_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
};

BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph,
           struct tcphdr *, th, u32, th_len)
{
#if IS_BUILTIN(CONFIG_IPV6)
        const u16 mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
                sizeof(struct ipv6hdr);
        u32 cookie;
        u16 mss;

        if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
                return -EINVAL;

        mss = tcp_parse_mss_option(th, 0) ?: mss_clamp;
        cookie = __cookie_v6_init_sequence(iph, th, &mss);

        return cookie | ((u64)mss << 32);
#else
        return -EPROTONOSUPPORT;
#endif
}

static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = {
        .func                = bpf_tcp_raw_gen_syncookie_ipv6,
        .gpl_only        = true, /* __cookie_v6_init_sequence() is GPL */
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg1_size        = sizeof(struct ipv6hdr),
        .arg2_type        = ARG_PTR_TO_MEM,
        .arg3_type        = ARG_CONST_SIZE_OR_ZERO,
};

BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph,
           struct tcphdr *, th)
{
        if (__cookie_v4_check(iph, th) > 0)
                return 0;

        return -EACCES;
}

static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = {
        .func                = bpf_tcp_raw_check_syncookie_ipv4,
        .gpl_only        = true, /* __cookie_v4_check is GPL */
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg1_size        = sizeof(struct iphdr),
        .arg2_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg2_size        = sizeof(struct tcphdr),
};

BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv6, struct ipv6hdr *, iph,
           struct tcphdr *, th)
{
#if IS_BUILTIN(CONFIG_IPV6)
        if (__cookie_v6_check(iph, th) > 0)
                return 0;

        return -EACCES;
#else
        return -EPROTONOSUPPORT;
#endif
}

static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = {
        .func                = bpf_tcp_raw_check_syncookie_ipv6,
        .gpl_only        = true, /* __cookie_v6_check is GPL */
        .pkt_access        = true,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg1_size        = sizeof(struct ipv6hdr),
        .arg2_type        = ARG_PTR_TO_FIXED_SIZE_MEM,
        .arg2_size        = sizeof(struct tcphdr),
};
#endif /* CONFIG_SYN_COOKIES */

#endif /* CONFIG_INET */

bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id)
{
        switch (func_id) {
        case BPF_FUNC_clone_redirect:
        case BPF_FUNC_l3_csum_replace:
        case BPF_FUNC_l4_csum_replace:
        case BPF_FUNC_lwt_push_encap:
        case BPF_FUNC_lwt_seg6_action:
        case BPF_FUNC_lwt_seg6_adjust_srh:
        case BPF_FUNC_lwt_seg6_store_bytes:
        case BPF_FUNC_msg_pop_data:
        case BPF_FUNC_msg_pull_data:
        case BPF_FUNC_msg_push_data:
        case BPF_FUNC_skb_adjust_room:
        case BPF_FUNC_skb_change_head:
        case BPF_FUNC_skb_change_proto:
        case BPF_FUNC_skb_change_tail:
        case BPF_FUNC_skb_pull_data:
        case BPF_FUNC_skb_store_bytes:
        case BPF_FUNC_skb_vlan_pop:
        case BPF_FUNC_skb_vlan_push:
        case BPF_FUNC_store_hdr_opt:
        case BPF_FUNC_xdp_adjust_head:
        case BPF_FUNC_xdp_adjust_meta:
        case BPF_FUNC_xdp_adjust_tail:
        /* tail-called program could call any of the above */
        case BPF_FUNC_tail_call:
                return true;
        default:
                return false;
        }
}

const struct bpf_func_proto bpf_event_output_data_proto __weak;
const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak;

static const struct bpf_func_proto *
sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_sock_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_sock_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_cg_sock_proto;
        case BPF_FUNC_ktime_get_coarse_ns:
                return &bpf_ktime_get_coarse_ns_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_bind:
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET4_CONNECT:
                case BPF_CGROUP_INET6_CONNECT:
                        return &bpf_bind_proto;
                default:
                        return NULL;
                }
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_sock_addr_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_sock_addr_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
#ifdef CONFIG_INET
        case BPF_FUNC_sk_lookup_tcp:
                return &bpf_sock_addr_sk_lookup_tcp_proto;
        case BPF_FUNC_sk_lookup_udp:
                return &bpf_sock_addr_sk_lookup_udp_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        case BPF_FUNC_skc_lookup_tcp:
                return &bpf_sock_addr_skc_lookup_tcp_proto;
#endif /* CONFIG_INET */
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
        case BPF_FUNC_setsockopt:
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET4_BIND:
                case BPF_CGROUP_INET6_BIND:
                case BPF_CGROUP_INET4_CONNECT:
                case BPF_CGROUP_INET6_CONNECT:
                case BPF_CGROUP_UNIX_CONNECT:
                case BPF_CGROUP_UDP4_RECVMSG:
                case BPF_CGROUP_UDP6_RECVMSG:
                case BPF_CGROUP_UNIX_RECVMSG:
                case BPF_CGROUP_UDP4_SENDMSG:
                case BPF_CGROUP_UDP6_SENDMSG:
                case BPF_CGROUP_UNIX_SENDMSG:
                case BPF_CGROUP_INET4_GETPEERNAME:
                case BPF_CGROUP_INET6_GETPEERNAME:
                case BPF_CGROUP_UNIX_GETPEERNAME:
                case BPF_CGROUP_INET4_GETSOCKNAME:
                case BPF_CGROUP_INET6_GETSOCKNAME:
                case BPF_CGROUP_UNIX_GETSOCKNAME:
                        return &bpf_sock_addr_setsockopt_proto;
                default:
                        return NULL;
                }
        case BPF_FUNC_getsockopt:
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET4_BIND:
                case BPF_CGROUP_INET6_BIND:
                case BPF_CGROUP_INET4_CONNECT:
                case BPF_CGROUP_INET6_CONNECT:
                case BPF_CGROUP_UNIX_CONNECT:
                case BPF_CGROUP_UDP4_RECVMSG:
                case BPF_CGROUP_UDP6_RECVMSG:
                case BPF_CGROUP_UNIX_RECVMSG:
                case BPF_CGROUP_UDP4_SENDMSG:
                case BPF_CGROUP_UDP6_SENDMSG:
                case BPF_CGROUP_UNIX_SENDMSG:
                case BPF_CGROUP_INET4_GETPEERNAME:
                case BPF_CGROUP_INET6_GETPEERNAME:
                case BPF_CGROUP_UNIX_GETPEERNAME:
                case BPF_CGROUP_INET4_GETSOCKNAME:
                case BPF_CGROUP_INET6_GETSOCKNAME:
                case BPF_CGROUP_UNIX_GETSOCKNAME:
                        return &bpf_sock_addr_getsockopt_proto;
                default:
                        return NULL;
                }
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_load_bytes:
                return &bpf_skb_load_bytes_proto;
        case BPF_FUNC_skb_load_bytes_relative:
                return &bpf_skb_load_bytes_relative_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_proto;
        case BPF_FUNC_get_socket_uid:
                return &bpf_get_socket_uid_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_skb_event_output_proto;
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

const struct bpf_func_proto bpf_sk_storage_get_proto __weak;
const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;

static const struct bpf_func_proto *
cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_sk_fullsock:
                return &bpf_sk_fullsock_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_skb_event_output_proto;
#ifdef CONFIG_SOCK_CGROUP_DATA
        case BPF_FUNC_skb_cgroup_id:
                return &bpf_skb_cgroup_id_proto;
        case BPF_FUNC_skb_ancestor_cgroup_id:
                return &bpf_skb_ancestor_cgroup_id_proto;
        case BPF_FUNC_sk_cgroup_id:
                return &bpf_sk_cgroup_id_proto;
        case BPF_FUNC_sk_ancestor_cgroup_id:
                return &bpf_sk_ancestor_cgroup_id_proto;
#endif
#ifdef CONFIG_INET
        case BPF_FUNC_sk_lookup_tcp:
                return &bpf_sk_lookup_tcp_proto;
        case BPF_FUNC_sk_lookup_udp:
                return &bpf_sk_lookup_udp_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        case BPF_FUNC_skc_lookup_tcp:
                return &bpf_skc_lookup_tcp_proto;
        case BPF_FUNC_tcp_sock:
                return &bpf_tcp_sock_proto;
        case BPF_FUNC_get_listener_sock:
                return &bpf_get_listener_sock_proto;
        case BPF_FUNC_skb_ecn_set_ce:
                return &bpf_skb_ecn_set_ce_proto;
#endif
        default:
                return sk_filter_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_store_bytes:
                return &bpf_skb_store_bytes_proto;
        case BPF_FUNC_skb_load_bytes:
                return &bpf_skb_load_bytes_proto;
        case BPF_FUNC_skb_load_bytes_relative:
                return &bpf_skb_load_bytes_relative_proto;
        case BPF_FUNC_skb_pull_data:
                return &bpf_skb_pull_data_proto;
        case BPF_FUNC_csum_diff:
                return &bpf_csum_diff_proto;
        case BPF_FUNC_csum_update:
                return &bpf_csum_update_proto;
        case BPF_FUNC_csum_level:
                return &bpf_csum_level_proto;
        case BPF_FUNC_l3_csum_replace:
                return &bpf_l3_csum_replace_proto;
        case BPF_FUNC_l4_csum_replace:
                return &bpf_l4_csum_replace_proto;
        case BPF_FUNC_clone_redirect:
                return &bpf_clone_redirect_proto;
        case BPF_FUNC_get_cgroup_classid:
                return &bpf_get_cgroup_classid_proto;
        case BPF_FUNC_skb_vlan_push:
                return &bpf_skb_vlan_push_proto;
        case BPF_FUNC_skb_vlan_pop:
                return &bpf_skb_vlan_pop_proto;
        case BPF_FUNC_skb_change_proto:
                return &bpf_skb_change_proto_proto;
        case BPF_FUNC_skb_change_type:
                return &bpf_skb_change_type_proto;
        case BPF_FUNC_skb_adjust_room:
                return &bpf_skb_adjust_room_proto;
        case BPF_FUNC_skb_change_tail:
                return &bpf_skb_change_tail_proto;
        case BPF_FUNC_skb_change_head:
                return &bpf_skb_change_head_proto;
        case BPF_FUNC_skb_get_tunnel_key:
                return &bpf_skb_get_tunnel_key_proto;
        case BPF_FUNC_skb_set_tunnel_key:
                return bpf_get_skb_set_tunnel_proto(func_id);
        case BPF_FUNC_skb_get_tunnel_opt:
                return &bpf_skb_get_tunnel_opt_proto;
        case BPF_FUNC_skb_set_tunnel_opt:
                return bpf_get_skb_set_tunnel_proto(func_id);
        case BPF_FUNC_redirect:
                return &bpf_redirect_proto;
        case BPF_FUNC_redirect_neigh:
                return &bpf_redirect_neigh_proto;
        case BPF_FUNC_redirect_peer:
                return &bpf_redirect_peer_proto;
        case BPF_FUNC_get_route_realm:
                return &bpf_get_route_realm_proto;
        case BPF_FUNC_get_hash_recalc:
                return &bpf_get_hash_recalc_proto;
        case BPF_FUNC_set_hash_invalid:
                return &bpf_set_hash_invalid_proto;
        case BPF_FUNC_set_hash:
                return &bpf_set_hash_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_skb_event_output_proto;
        case BPF_FUNC_get_smp_processor_id:
                return &bpf_get_smp_processor_id_proto;
        case BPF_FUNC_skb_under_cgroup:
                return &bpf_skb_under_cgroup_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_proto;
        case BPF_FUNC_get_socket_uid:
                return &bpf_get_socket_uid_proto;
        case BPF_FUNC_fib_lookup:
                return &bpf_skb_fib_lookup_proto;
        case BPF_FUNC_check_mtu:
                return &bpf_skb_check_mtu_proto;
        case BPF_FUNC_sk_fullsock:
                return &bpf_sk_fullsock_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
#ifdef CONFIG_XFRM
        case BPF_FUNC_skb_get_xfrm_state:
                return &bpf_skb_get_xfrm_state_proto;
#endif
#ifdef CONFIG_CGROUP_NET_CLASSID
        case BPF_FUNC_skb_cgroup_classid:
                return &bpf_skb_cgroup_classid_proto;
#endif
#ifdef CONFIG_SOCK_CGROUP_DATA
        case BPF_FUNC_skb_cgroup_id:
                return &bpf_skb_cgroup_id_proto;
        case BPF_FUNC_skb_ancestor_cgroup_id:
                return &bpf_skb_ancestor_cgroup_id_proto;
#endif
#ifdef CONFIG_INET
        case BPF_FUNC_sk_lookup_tcp:
                return &bpf_tc_sk_lookup_tcp_proto;
        case BPF_FUNC_sk_lookup_udp:
                return &bpf_tc_sk_lookup_udp_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        case BPF_FUNC_tcp_sock:
                return &bpf_tcp_sock_proto;
        case BPF_FUNC_get_listener_sock:
                return &bpf_get_listener_sock_proto;
        case BPF_FUNC_skc_lookup_tcp:
                return &bpf_tc_skc_lookup_tcp_proto;
        case BPF_FUNC_tcp_check_syncookie:
                return &bpf_tcp_check_syncookie_proto;
        case BPF_FUNC_skb_ecn_set_ce:
                return &bpf_skb_ecn_set_ce_proto;
        case BPF_FUNC_tcp_gen_syncookie:
                return &bpf_tcp_gen_syncookie_proto;
        case BPF_FUNC_sk_assign:
                return &bpf_sk_assign_proto;
        case BPF_FUNC_skb_set_tstamp:
                return &bpf_skb_set_tstamp_proto;
#ifdef CONFIG_SYN_COOKIES
        case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
                return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
        case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
                return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
        case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
                return &bpf_tcp_raw_check_syncookie_ipv4_proto;
        case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
                return &bpf_tcp_raw_check_syncookie_ipv6_proto;
#endif
#endif
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_perf_event_output:
                return &bpf_xdp_event_output_proto;
        case BPF_FUNC_get_smp_processor_id:
                return &bpf_get_smp_processor_id_proto;
        case BPF_FUNC_csum_diff:
                return &bpf_csum_diff_proto;
        case BPF_FUNC_xdp_adjust_head:
                return &bpf_xdp_adjust_head_proto;
        case BPF_FUNC_xdp_adjust_meta:
                return &bpf_xdp_adjust_meta_proto;
        case BPF_FUNC_redirect:
                return &bpf_xdp_redirect_proto;
        case BPF_FUNC_redirect_map:
                return &bpf_xdp_redirect_map_proto;
        case BPF_FUNC_xdp_adjust_tail:
                return &bpf_xdp_adjust_tail_proto;
        case BPF_FUNC_xdp_get_buff_len:
                return &bpf_xdp_get_buff_len_proto;
        case BPF_FUNC_xdp_load_bytes:
                return &bpf_xdp_load_bytes_proto;
        case BPF_FUNC_xdp_store_bytes:
                return &bpf_xdp_store_bytes_proto;
        case BPF_FUNC_fib_lookup:
                return &bpf_xdp_fib_lookup_proto;
        case BPF_FUNC_check_mtu:
                return &bpf_xdp_check_mtu_proto;
#ifdef CONFIG_INET
        case BPF_FUNC_sk_lookup_udp:
                return &bpf_xdp_sk_lookup_udp_proto;
        case BPF_FUNC_sk_lookup_tcp:
                return &bpf_xdp_sk_lookup_tcp_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        case BPF_FUNC_skc_lookup_tcp:
                return &bpf_xdp_skc_lookup_tcp_proto;
        case BPF_FUNC_tcp_check_syncookie:
                return &bpf_tcp_check_syncookie_proto;
        case BPF_FUNC_tcp_gen_syncookie:
                return &bpf_tcp_gen_syncookie_proto;
#ifdef CONFIG_SYN_COOKIES
        case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
                return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
        case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
                return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
        case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
                return &bpf_tcp_raw_check_syncookie_ipv4_proto;
        case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
                return &bpf_tcp_raw_check_syncookie_ipv6_proto;
#endif
#endif
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }

#if IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES)
        /* The nf_conn___init type is used in the NF_CONNTRACK kfuncs. The
         * kfuncs are defined in two different modules, and we want to be able
         * to use them interchangeably with the same BTF type ID. Because modules
         * can't de-duplicate BTF IDs between each other, we need the type to be
         * referenced in the vmlinux BTF or the verifier will get confused about
         * the different types. So we add this dummy type reference which will
         * be included in vmlinux BTF, allowing both modules to refer to the
         * same type ID.
         */
        BTF_TYPE_EMIT(struct nf_conn___init);
#endif
}

const struct bpf_func_proto bpf_sock_map_update_proto __weak;
const struct bpf_func_proto bpf_sock_hash_update_proto __weak;

static const struct bpf_func_proto *
sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func_proto;

        func_proto = cgroup_common_func_proto(func_id, prog);
        if (func_proto)
                return func_proto;

        switch (func_id) {
        case BPF_FUNC_setsockopt:
                return &bpf_sock_ops_setsockopt_proto;
        case BPF_FUNC_getsockopt:
                return &bpf_sock_ops_getsockopt_proto;
        case BPF_FUNC_sock_ops_cb_flags_set:
                return &bpf_sock_ops_cb_flags_set_proto;
        case BPF_FUNC_sock_map_update:
                return &bpf_sock_map_update_proto;
        case BPF_FUNC_sock_hash_update:
                return &bpf_sock_hash_update_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_sock_ops_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_sock_ops_proto;
#ifdef CONFIG_INET
        case BPF_FUNC_load_hdr_opt:
                return &bpf_sock_ops_load_hdr_opt_proto;
        case BPF_FUNC_store_hdr_opt:
                return &bpf_sock_ops_store_hdr_opt_proto;
        case BPF_FUNC_reserve_hdr_opt:
                return &bpf_sock_ops_reserve_hdr_opt_proto;
        case BPF_FUNC_tcp_sock:
                return &bpf_tcp_sock_proto;
#endif /* CONFIG_INET */
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

const struct bpf_func_proto bpf_msg_redirect_map_proto __weak;
const struct bpf_func_proto bpf_msg_redirect_hash_proto __weak;

static const struct bpf_func_proto *
sk_msg_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_msg_redirect_map:
                return &bpf_msg_redirect_map_proto;
        case BPF_FUNC_msg_redirect_hash:
                return &bpf_msg_redirect_hash_proto;
        case BPF_FUNC_msg_apply_bytes:
                return &bpf_msg_apply_bytes_proto;
        case BPF_FUNC_msg_cork_bytes:
                return &bpf_msg_cork_bytes_proto;
        case BPF_FUNC_msg_pull_data:
                return &bpf_msg_pull_data_proto;
        case BPF_FUNC_msg_push_data:
                return &bpf_msg_push_data_proto;
        case BPF_FUNC_msg_pop_data:
                return &bpf_msg_pop_data_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        case BPF_FUNC_sk_storage_get:
                return &bpf_sk_storage_get_proto;
        case BPF_FUNC_sk_storage_delete:
                return &bpf_sk_storage_delete_proto;
        case BPF_FUNC_get_netns_cookie:
                return &bpf_get_netns_cookie_sk_msg_proto;
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

const struct bpf_func_proto bpf_sk_redirect_map_proto __weak;
const struct bpf_func_proto bpf_sk_redirect_hash_proto __weak;

static const struct bpf_func_proto *
sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_store_bytes:
                return &bpf_skb_store_bytes_proto;
        case BPF_FUNC_skb_load_bytes:
                return &bpf_skb_load_bytes_proto;
        case BPF_FUNC_skb_pull_data:
                return &sk_skb_pull_data_proto;
        case BPF_FUNC_skb_change_tail:
                return &sk_skb_change_tail_proto;
        case BPF_FUNC_skb_change_head:
                return &sk_skb_change_head_proto;
        case BPF_FUNC_skb_adjust_room:
                return &sk_skb_adjust_room_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_cookie_proto;
        case BPF_FUNC_get_socket_uid:
                return &bpf_get_socket_uid_proto;
        case BPF_FUNC_sk_redirect_map:
                return &bpf_sk_redirect_map_proto;
        case BPF_FUNC_sk_redirect_hash:
                return &bpf_sk_redirect_hash_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_skb_event_output_proto;
#ifdef CONFIG_INET
        case BPF_FUNC_sk_lookup_tcp:
                return &bpf_sk_lookup_tcp_proto;
        case BPF_FUNC_sk_lookup_udp:
                return &bpf_sk_lookup_udp_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        case BPF_FUNC_skc_lookup_tcp:
                return &bpf_skc_lookup_tcp_proto;
#endif
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
flow_dissector_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_load_bytes:
                return &bpf_flow_dissector_load_bytes_proto;
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
lwt_out_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_load_bytes:
                return &bpf_skb_load_bytes_proto;
        case BPF_FUNC_skb_pull_data:
                return &bpf_skb_pull_data_proto;
        case BPF_FUNC_csum_diff:
                return &bpf_csum_diff_proto;
        case BPF_FUNC_get_cgroup_classid:
                return &bpf_get_cgroup_classid_proto;
        case BPF_FUNC_get_route_realm:
                return &bpf_get_route_realm_proto;
        case BPF_FUNC_get_hash_recalc:
                return &bpf_get_hash_recalc_proto;
        case BPF_FUNC_perf_event_output:
                return &bpf_skb_event_output_proto;
        case BPF_FUNC_get_smp_processor_id:
                return &bpf_get_smp_processor_id_proto;
        case BPF_FUNC_skb_under_cgroup:
                return &bpf_skb_under_cgroup_proto;
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
lwt_in_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_lwt_push_encap:
                return &bpf_lwt_in_push_encap_proto;
        default:
                return lwt_out_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
lwt_xmit_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_skb_get_tunnel_key:
                return &bpf_skb_get_tunnel_key_proto;
        case BPF_FUNC_skb_set_tunnel_key:
                return bpf_get_skb_set_tunnel_proto(func_id);
        case BPF_FUNC_skb_get_tunnel_opt:
                return &bpf_skb_get_tunnel_opt_proto;
        case BPF_FUNC_skb_set_tunnel_opt:
                return bpf_get_skb_set_tunnel_proto(func_id);
        case BPF_FUNC_redirect:
                return &bpf_redirect_proto;
        case BPF_FUNC_clone_redirect:
                return &bpf_clone_redirect_proto;
        case BPF_FUNC_skb_change_tail:
                return &bpf_skb_change_tail_proto;
        case BPF_FUNC_skb_change_head:
                return &bpf_skb_change_head_proto;
        case BPF_FUNC_skb_store_bytes:
                return &bpf_skb_store_bytes_proto;
        case BPF_FUNC_csum_update:
                return &bpf_csum_update_proto;
        case BPF_FUNC_csum_level:
                return &bpf_csum_level_proto;
        case BPF_FUNC_l3_csum_replace:
                return &bpf_l3_csum_replace_proto;
        case BPF_FUNC_l4_csum_replace:
                return &bpf_l4_csum_replace_proto;
        case BPF_FUNC_set_hash_invalid:
                return &bpf_set_hash_invalid_proto;
        case BPF_FUNC_lwt_push_encap:
                return &bpf_lwt_xmit_push_encap_proto;
        default:
                return lwt_out_func_proto(func_id, prog);
        }
}

static const struct bpf_func_proto *
lwt_seg6local_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
#if IS_ENABLED(CONFIG_IPV6_SEG6_BPF)
        case BPF_FUNC_lwt_seg6_store_bytes:
                return &bpf_lwt_seg6_store_bytes_proto;
        case BPF_FUNC_lwt_seg6_action:
                return &bpf_lwt_seg6_action_proto;
        case BPF_FUNC_lwt_seg6_adjust_srh:
                return &bpf_lwt_seg6_adjust_srh_proto;
#endif
        default:
                return lwt_out_func_proto(func_id, prog);
        }
}

static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type,
                                    const struct bpf_prog *prog,
                                    struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct __sk_buff))
                return false;

        /* The verifier guarantees that size > 0. */
        if (off % size != 0)
                return false;

        switch (off) {
        case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
                if (off + size > offsetofend(struct __sk_buff, cb[4]))
                        return false;
                break;
        case bpf_ctx_range(struct __sk_buff, data):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, data_end):
                if (info->is_ldsx || size != size_default)
                        return false;
                break;
        case bpf_ctx_range_till(struct __sk_buff, remote_ip6[0], remote_ip6[3]):
        case bpf_ctx_range_till(struct __sk_buff, local_ip6[0], local_ip6[3]):
        case bpf_ctx_range_till(struct __sk_buff, remote_ip4, remote_ip4):
        case bpf_ctx_range_till(struct __sk_buff, local_ip4, local_ip4):
                if (size != size_default)
                        return false;
                break;
        case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                return false;
        case bpf_ctx_range(struct __sk_buff, hwtstamp):
                if (type == BPF_WRITE || size != sizeof(__u64))
                        return false;
                break;
        case bpf_ctx_range(struct __sk_buff, tstamp):
                if (size != sizeof(__u64))
                        return false;
                break;
        case offsetof(struct __sk_buff, sk):
                if (type == BPF_WRITE || size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
                break;
        case offsetof(struct __sk_buff, tstamp_type):
                return false;
        case offsetofend(struct __sk_buff, tstamp_type) ... offsetof(struct __sk_buff, hwtstamp) - 1:
                /* Explicitly prohibit access to padding in __sk_buff. */
                return false;
        default:
                /* Only narrow read access allowed for now. */
                if (type == BPF_WRITE) {
                        if (size != size_default)
                                return false;
                } else {
                        bpf_ctx_record_field_size(info, size_default);
                        if (!bpf_ctx_narrow_access_ok(off, size, size_default))
                                return false;
                }
        }

        return true;
}

static bool sk_filter_is_valid_access(int off, int size,
                                      enum bpf_access_type type,
                                      const struct bpf_prog *prog,
                                      struct bpf_insn_access_aux *info)
{
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range(struct __sk_buff, data):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, data_end):
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
        case bpf_ctx_range(struct __sk_buff, tstamp):
        case bpf_ctx_range(struct __sk_buff, wire_len):
        case bpf_ctx_range(struct __sk_buff, hwtstamp):
                return false;
        }

        if (type == BPF_WRITE) {
                switch (off) {
                case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
                        break;
                default:
                        return false;
                }
        }

        return bpf_skb_is_valid_access(off, size, type, prog, info);
}

static bool cg_skb_is_valid_access(int off, int size,
                                   enum bpf_access_type type,
                                   const struct bpf_prog *prog,
                                   struct bpf_insn_access_aux *info)
{
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, wire_len):
                return false;
        case bpf_ctx_range(struct __sk_buff, data):
        case bpf_ctx_range(struct __sk_buff, data_end):
                if (!bpf_token_capable(prog->aux->token, CAP_BPF))
                        return false;
                break;
        }

        if (type == BPF_WRITE) {
                switch (off) {
                case bpf_ctx_range(struct __sk_buff, mark):
                case bpf_ctx_range(struct __sk_buff, priority):
                case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
                        break;
                case bpf_ctx_range(struct __sk_buff, tstamp):
                        if (!bpf_token_capable(prog->aux->token, CAP_BPF))
                                return false;
                        break;
                default:
                        return false;
                }
        }

        switch (off) {
        case bpf_ctx_range(struct __sk_buff, data):
                info->reg_type = PTR_TO_PACKET;
                break;
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
        }

        return bpf_skb_is_valid_access(off, size, type, prog, info);
}

static bool lwt_is_valid_access(int off, int size,
                                enum bpf_access_type type,
                                const struct bpf_prog *prog,
                                struct bpf_insn_access_aux *info)
{
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, tstamp):
        case bpf_ctx_range(struct __sk_buff, wire_len):
        case bpf_ctx_range(struct __sk_buff, hwtstamp):
                return false;
        }

        if (type == BPF_WRITE) {
                switch (off) {
                case bpf_ctx_range(struct __sk_buff, mark):
                case bpf_ctx_range(struct __sk_buff, priority):
                case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
                        break;
                default:
                        return false;
                }
        }

        switch (off) {
        case bpf_ctx_range(struct __sk_buff, data):
                info->reg_type = PTR_TO_PACKET;
                break;
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
        }

        return bpf_skb_is_valid_access(off, size, type, prog, info);
}

/* Attach type specific accesses */
static bool __sock_filter_check_attach_type(int off,
                                            enum bpf_access_type access_type,
                                            enum bpf_attach_type attach_type)
{
        switch (off) {
        case offsetof(struct bpf_sock, bound_dev_if):
        case offsetof(struct bpf_sock, mark):
        case offsetof(struct bpf_sock, priority):
                switch (attach_type) {
                case BPF_CGROUP_INET_SOCK_CREATE:
                case BPF_CGROUP_INET_SOCK_RELEASE:
                        goto full_access;
                default:
                        return false;
                }
        case bpf_ctx_range(struct bpf_sock, src_ip4):
                switch (attach_type) {
                case BPF_CGROUP_INET4_POST_BIND:
                        goto read_only;
                default:
                        return false;
                }
        case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
                switch (attach_type) {
                case BPF_CGROUP_INET6_POST_BIND:
                        goto read_only;
                default:
                        return false;
                }
        case bpf_ctx_range(struct bpf_sock, src_port):
                switch (attach_type) {
                case BPF_CGROUP_INET4_POST_BIND:
                case BPF_CGROUP_INET6_POST_BIND:
                        goto read_only;
                default:
                        return false;
                }
        }
read_only:
        return access_type == BPF_READ;
full_access:
        return true;
}

bool bpf_sock_common_is_valid_access(int off, int size,
                                     enum bpf_access_type type,
                                     struct bpf_insn_access_aux *info)
{
        switch (off) {
        case bpf_ctx_range_till(struct bpf_sock, type, priority):
                return false;
        default:
                return bpf_sock_is_valid_access(off, size, type, info);
        }
}

bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type,
                              struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);
        int field_size;

        if (off < 0 || off >= sizeof(struct bpf_sock))
                return false;
        if (off % size != 0)
                return false;

        switch (off) {
        case offsetof(struct bpf_sock, state):
        case offsetof(struct bpf_sock, family):
        case offsetof(struct bpf_sock, type):
        case offsetof(struct bpf_sock, protocol):
        case offsetof(struct bpf_sock, src_port):
        case offsetof(struct bpf_sock, rx_queue_mapping):
        case bpf_ctx_range(struct bpf_sock, src_ip4):
        case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
        case bpf_ctx_range(struct bpf_sock, dst_ip4):
        case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
                bpf_ctx_record_field_size(info, size_default);
                return bpf_ctx_narrow_access_ok(off, size, size_default);
        case bpf_ctx_range(struct bpf_sock, dst_port):
                field_size = size == size_default ?
                        size_default : sizeof_field(struct bpf_sock, dst_port);
                bpf_ctx_record_field_size(info, field_size);
                return bpf_ctx_narrow_access_ok(off, size, field_size);
        case offsetofend(struct bpf_sock, dst_port) ...
             offsetof(struct bpf_sock, dst_ip4) - 1:
                return false;
        }

        return size == size_default;
}

static bool sock_filter_is_valid_access(int off, int size,
                                        enum bpf_access_type type,
                                        const struct bpf_prog *prog,
                                        struct bpf_insn_access_aux *info)
{
        if (!bpf_sock_is_valid_access(off, size, type, info))
                return false;
        return __sock_filter_check_attach_type(off, type,
                                               prog->expected_attach_type);
}

static int bpf_noop_prologue(struct bpf_insn *insn_buf, bool direct_write,
                             const struct bpf_prog *prog)
{
        /* Neither direct read nor direct write requires any preliminary
         * action.
         */
        return 0;
}

static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write,
                                const struct bpf_prog *prog, int drop_verdict)
{
        struct bpf_insn *insn = insn_buf;

        if (!direct_write)
                return 0;

        /* if (!skb->cloned)
         *       goto start;
         *
         * (Fast-path, otherwise approximation that we might be
         *  a clone, do the rest in helper.)
         */
        *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET);
        *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK);
        *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7);

        /* ret = bpf_skb_pull_data(skb, 0); */
        *insn++ = BPF_MOV64_REG(BPF_REG_6, BPF_REG_1);
        *insn++ = BPF_ALU64_REG(BPF_XOR, BPF_REG_2, BPF_REG_2);
        *insn++ = BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
                               BPF_FUNC_skb_pull_data);
        /* if (!ret)
         *      goto restore;
         * return TC_ACT_SHOT;
         */
        *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2);
        *insn++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_0, drop_verdict);
        *insn++ = BPF_EXIT_INSN();

        /* restore: */
        *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_6);
        /* start: */
        *insn++ = prog->insnsi[0];

        return insn - insn_buf;
}

static int bpf_gen_ld_abs(const struct bpf_insn *orig,
                          struct bpf_insn *insn_buf)
{
        bool indirect = BPF_MODE(orig->code) == BPF_IND;
        struct bpf_insn *insn = insn_buf;

        if (!indirect) {
                *insn++ = BPF_MOV64_IMM(BPF_REG_2, orig->imm);
        } else {
                *insn++ = BPF_MOV64_REG(BPF_REG_2, orig->src_reg);
                if (orig->imm)
                        *insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, orig->imm);
        }
        /* We're guaranteed here that CTX is in R6. */
        *insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_CTX);

        switch (BPF_SIZE(orig->code)) {
        case BPF_B:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_8_no_cache);
                break;
        case BPF_H:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_16_no_cache);
                break;
        case BPF_W:
                *insn++ = BPF_EMIT_CALL(bpf_skb_load_helper_32_no_cache);
                break;
        }

        *insn++ = BPF_JMP_IMM(BPF_JSGE, BPF_REG_0, 0, 2);
        *insn++ = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0);
        *insn++ = BPF_EXIT_INSN();

        return insn - insn_buf;
}

static int tc_cls_act_prologue(struct bpf_insn *insn_buf, bool direct_write,
                               const struct bpf_prog *prog)
{
        return bpf_unclone_prologue(insn_buf, direct_write, prog, TC_ACT_SHOT);
}

static bool tc_cls_act_is_valid_access(int off, int size,
                                       enum bpf_access_type type,
                                       const struct bpf_prog *prog,
                                       struct bpf_insn_access_aux *info)
{
        if (type == BPF_WRITE) {
                switch (off) {
                case bpf_ctx_range(struct __sk_buff, mark):
                case bpf_ctx_range(struct __sk_buff, tc_index):
                case bpf_ctx_range(struct __sk_buff, priority):
                case bpf_ctx_range(struct __sk_buff, tc_classid):
                case bpf_ctx_range_till(struct __sk_buff, cb[0], cb[4]):
                case bpf_ctx_range(struct __sk_buff, tstamp):
                case bpf_ctx_range(struct __sk_buff, queue_mapping):
                        break;
                default:
                        return false;
                }
        }

        switch (off) {
        case bpf_ctx_range(struct __sk_buff, data):
                info->reg_type = PTR_TO_PACKET;
                break;
        case bpf_ctx_range(struct __sk_buff, data_meta):
                info->reg_type = PTR_TO_PACKET_META;
                break;
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
        case bpf_ctx_range_till(struct __sk_buff, family, local_port):
                return false;
        case offsetof(struct __sk_buff, tstamp_type):
                /* The convert_ctx_access() on reading and writing
                 * __sk_buff->tstamp depends on whether the bpf prog
                 * has used __sk_buff->tstamp_type or not.
                 * Thus, we need to set prog->tstamp_type_access
                 * earlier during is_valid_access() here.
                 */
                ((struct bpf_prog *)prog)->tstamp_type_access = 1;
                return size == sizeof(__u8);
        }

        return bpf_skb_is_valid_access(off, size, type, prog, info);
}

DEFINE_MUTEX(nf_conn_btf_access_lock);
EXPORT_SYMBOL_GPL(nf_conn_btf_access_lock);

int (*nfct_btf_struct_access)(struct bpf_verifier_log *log,
                              const struct bpf_reg_state *reg,
                              int off, int size);
EXPORT_SYMBOL_GPL(nfct_btf_struct_access);

static int tc_cls_act_btf_struct_access(struct bpf_verifier_log *log,
                                        const struct bpf_reg_state *reg,
                                        int off, int size)
{
        int ret = -EACCES;

        mutex_lock(&nf_conn_btf_access_lock);
        if (nfct_btf_struct_access)
                ret = nfct_btf_struct_access(log, reg, off, size);
        mutex_unlock(&nf_conn_btf_access_lock);

        return ret;
}

static bool __is_valid_xdp_access(int off, int size)
{
        if (off < 0 || off >= sizeof(struct xdp_md))
                return false;
        if (off % size != 0)
                return false;
        if (size != sizeof(__u32))
                return false;

        return true;
}

static bool xdp_is_valid_access(int off, int size,
                                enum bpf_access_type type,
                                const struct bpf_prog *prog,
                                struct bpf_insn_access_aux *info)
{
        if (prog->expected_attach_type != BPF_XDP_DEVMAP) {
                switch (off) {
                case offsetof(struct xdp_md, egress_ifindex):
                        return false;
                }
        }

        if (type == BPF_WRITE) {
                if (bpf_prog_is_offloaded(prog->aux)) {
                        switch (off) {
                        case offsetof(struct xdp_md, rx_queue_index):
                                return __is_valid_xdp_access(off, size);
                        }
                }
                return false;
        } else {
                switch (off) {
                case offsetof(struct xdp_md, data_meta):
                case offsetof(struct xdp_md, data):
                case offsetof(struct xdp_md, data_end):
                        if (info->is_ldsx)
                                return false;
                }
        }

        switch (off) {
        case offsetof(struct xdp_md, data):
                info->reg_type = PTR_TO_PACKET;
                break;
        case offsetof(struct xdp_md, data_meta):
                info->reg_type = PTR_TO_PACKET_META;
                break;
        case offsetof(struct xdp_md, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
        }

        return __is_valid_xdp_access(off, size);
}

void bpf_warn_invalid_xdp_action(const struct net_device *dev,
                                 const struct bpf_prog *prog, u32 act)
{
        const u32 act_max = XDP_REDIRECT;

        pr_warn_once("%s XDP return value %u on prog %s (id %d) dev %s, expect packet loss!\n",
                     act > act_max ? "Illegal" : "Driver unsupported",
                     act, prog->aux->name, prog->aux->id, dev ? dev->name : "N/A");
}
EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);

static int xdp_btf_struct_access(struct bpf_verifier_log *log,
                                 const struct bpf_reg_state *reg,
                                 int off, int size)
{
        int ret = -EACCES;

        mutex_lock(&nf_conn_btf_access_lock);
        if (nfct_btf_struct_access)
                ret = nfct_btf_struct_access(log, reg, off, size);
        mutex_unlock(&nf_conn_btf_access_lock);

        return ret;
}

static bool sock_addr_is_valid_access(int off, int size,
                                      enum bpf_access_type type,
                                      const struct bpf_prog *prog,
                                      struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct bpf_sock_addr))
                return false;
        if (off % size != 0)
                return false;

        /* Disallow access to fields not belonging to the attach type's address
         * family.
         */
        switch (off) {
        case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET4_BIND:
                case BPF_CGROUP_INET4_CONNECT:
                case BPF_CGROUP_INET4_GETPEERNAME:
                case BPF_CGROUP_INET4_GETSOCKNAME:
                case BPF_CGROUP_UDP4_SENDMSG:
                case BPF_CGROUP_UDP4_RECVMSG:
                        break;
                default:
                        return false;
                }
                break;
        case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_INET6_BIND:
                case BPF_CGROUP_INET6_CONNECT:
                case BPF_CGROUP_INET6_GETPEERNAME:
                case BPF_CGROUP_INET6_GETSOCKNAME:
                case BPF_CGROUP_UDP6_SENDMSG:
                case BPF_CGROUP_UDP6_RECVMSG:
                        break;
                default:
                        return false;
                }
                break;
        case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_UDP4_SENDMSG:
                        break;
                default:
                        return false;
                }
                break;
        case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
                                msg_src_ip6[3]):
                switch (prog->expected_attach_type) {
                case BPF_CGROUP_UDP6_SENDMSG:
                        break;
                default:
                        return false;
                }
                break;
        }

        switch (off) {
        case bpf_ctx_range(struct bpf_sock_addr, user_ip4):
        case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
        case bpf_ctx_range(struct bpf_sock_addr, msg_src_ip4):
        case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
                                msg_src_ip6[3]):
        case bpf_ctx_range(struct bpf_sock_addr, user_port):
                if (type == BPF_READ) {
                        bpf_ctx_record_field_size(info, size_default);

                        if (bpf_ctx_wide_access_ok(off, size,
                                                   struct bpf_sock_addr,
                                                   user_ip6))
                                return true;

                        if (bpf_ctx_wide_access_ok(off, size,
                                                   struct bpf_sock_addr,
                                                   msg_src_ip6))
                                return true;

                        if (!bpf_ctx_narrow_access_ok(off, size, size_default))
                                return false;
                } else {
                        if (bpf_ctx_wide_access_ok(off, size,
                                                   struct bpf_sock_addr,
                                                   user_ip6))
                                return true;

                        if (bpf_ctx_wide_access_ok(off, size,
                                                   struct bpf_sock_addr,
                                                   msg_src_ip6))
                                return true;

                        if (size != size_default)
                                return false;
                }
                break;
        case offsetof(struct bpf_sock_addr, sk):
                if (type != BPF_READ)
                        return false;
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_SOCKET;
                break;
        default:
                if (type == BPF_READ) {
                        if (size != size_default)
                                return false;
                } else {
                        return false;
                }
        }

        return true;
}

static bool sock_ops_is_valid_access(int off, int size,
                                     enum bpf_access_type type,
                                     const struct bpf_prog *prog,
                                     struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct bpf_sock_ops))
                return false;

        /* The verifier guarantees that size > 0. */
        if (off % size != 0)
                return false;

        if (type == BPF_WRITE) {
                switch (off) {
                case offsetof(struct bpf_sock_ops, reply):
                case offsetof(struct bpf_sock_ops, sk_txhash):
                        if (size != size_default)
                                return false;
                        break;
                default:
                        return false;
                }
        } else {
                switch (off) {
                case bpf_ctx_range_till(struct bpf_sock_ops, bytes_received,
                                        bytes_acked):
                        if (size != sizeof(__u64))
                                return false;
                        break;
                case offsetof(struct bpf_sock_ops, sk):
                        if (size != sizeof(__u64))
                                return false;
                        info->reg_type = PTR_TO_SOCKET_OR_NULL;
                        break;
                case offsetof(struct bpf_sock_ops, skb_data):
                        if (size != sizeof(__u64))
                                return false;
                        info->reg_type = PTR_TO_PACKET;
                        break;
                case offsetof(struct bpf_sock_ops, skb_data_end):
                        if (size != sizeof(__u64))
                                return false;
                        info->reg_type = PTR_TO_PACKET_END;
                        break;
                case offsetof(struct bpf_sock_ops, skb_tcp_flags):
                        bpf_ctx_record_field_size(info, size_default);
                        return bpf_ctx_narrow_access_ok(off, size,
                                                        size_default);
                case offsetof(struct bpf_sock_ops, skb_hwtstamp):
                        if (size != sizeof(__u64))
                                return false;
                        break;
                default:
                        if (size != size_default)
                                return false;
                        break;
                }
        }

        return true;
}

static int sk_skb_prologue(struct bpf_insn *insn_buf, bool direct_write,
                           const struct bpf_prog *prog)
{
        return bpf_unclone_prologue(insn_buf, direct_write, prog, SK_DROP);
}

static bool sk_skb_is_valid_access(int off, int size,
                                   enum bpf_access_type type,
                                   const struct bpf_prog *prog,
                                   struct bpf_insn_access_aux *info)
{
        switch (off) {
        case bpf_ctx_range(struct __sk_buff, tc_classid):
        case bpf_ctx_range(struct __sk_buff, data_meta):
        case bpf_ctx_range(struct __sk_buff, tstamp):
        case bpf_ctx_range(struct __sk_buff, wire_len):
        case bpf_ctx_range(struct __sk_buff, hwtstamp):
                return false;
        }

        if (type == BPF_WRITE) {
                switch (off) {
                case bpf_ctx_range(struct __sk_buff, tc_index):
                case bpf_ctx_range(struct __sk_buff, priority):
                        break;
                default:
                        return false;
                }
        }

        switch (off) {
        case bpf_ctx_range(struct __sk_buff, mark):
                return false;
        case bpf_ctx_range(struct __sk_buff, data):
                info->reg_type = PTR_TO_PACKET;
                break;
        case bpf_ctx_range(struct __sk_buff, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                break;
        }

        return bpf_skb_is_valid_access(off, size, type, prog, info);
}

static bool sk_msg_is_valid_access(int off, int size,
                                   enum bpf_access_type type,
                                   const struct bpf_prog *prog,
                                   struct bpf_insn_access_aux *info)
{
        if (type == BPF_WRITE)
                return false;

        if (off % size != 0)
                return false;

        switch (off) {
        case offsetof(struct sk_msg_md, data):
                info->reg_type = PTR_TO_PACKET;
                if (size != sizeof(__u64))
                        return false;
                break;
        case offsetof(struct sk_msg_md, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                if (size != sizeof(__u64))
                        return false;
                break;
        case offsetof(struct sk_msg_md, sk):
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_SOCKET;
                break;
        case bpf_ctx_range(struct sk_msg_md, family):
        case bpf_ctx_range(struct sk_msg_md, remote_ip4):
        case bpf_ctx_range(struct sk_msg_md, local_ip4):
        case bpf_ctx_range_till(struct sk_msg_md, remote_ip6[0], remote_ip6[3]):
        case bpf_ctx_range_till(struct sk_msg_md, local_ip6[0], local_ip6[3]):
        case bpf_ctx_range(struct sk_msg_md, remote_port):
        case bpf_ctx_range(struct sk_msg_md, local_port):
        case bpf_ctx_range(struct sk_msg_md, size):
                if (size != sizeof(__u32))
                        return false;
                break;
        default:
                return false;
        }
        return true;
}

static bool flow_dissector_is_valid_access(int off, int size,
                                           enum bpf_access_type type,
                                           const struct bpf_prog *prog,
                                           struct bpf_insn_access_aux *info)
{
        const int size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct __sk_buff))
                return false;

        if (type == BPF_WRITE)
                return false;

        switch (off) {
        case bpf_ctx_range(struct __sk_buff, data):
                if (info->is_ldsx || size != size_default)
                        return false;
                info->reg_type = PTR_TO_PACKET;
                return true;
        case bpf_ctx_range(struct __sk_buff, data_end):
                if (info->is_ldsx || size != size_default)
                        return false;
                info->reg_type = PTR_TO_PACKET_END;
                return true;
        case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
                if (size != sizeof(__u64))
                        return false;
                info->reg_type = PTR_TO_FLOW_KEYS;
                return true;
        default:
                return false;
        }
}

static u32 flow_dissector_convert_ctx_access(enum bpf_access_type type,
                                             const struct bpf_insn *si,
                                             struct bpf_insn *insn_buf,
                                             struct bpf_prog *prog,
                                             u32 *target_size)

{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct __sk_buff, data):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_flow_dissector, data));
                break;

        case offsetof(struct __sk_buff, data_end):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, data_end),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_flow_dissector, data_end));
                break;

        case offsetof(struct __sk_buff, flow_keys):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_flow_dissector, flow_keys),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_flow_dissector, flow_keys));
                break;
        }

        return insn - insn_buf;
}

static struct bpf_insn *bpf_convert_tstamp_type_read(const struct bpf_insn *si,
                                                     struct bpf_insn *insn)
{
        __u8 value_reg = si->dst_reg;
        __u8 skb_reg = si->src_reg;
        BUILD_BUG_ON(__SKB_CLOCK_MAX != (int)BPF_SKB_CLOCK_TAI);
        BUILD_BUG_ON(SKB_CLOCK_REALTIME != (int)BPF_SKB_CLOCK_REALTIME);
        BUILD_BUG_ON(SKB_CLOCK_MONOTONIC != (int)BPF_SKB_CLOCK_MONOTONIC);
        BUILD_BUG_ON(SKB_CLOCK_TAI != (int)BPF_SKB_CLOCK_TAI);
        *insn++ = BPF_LDX_MEM(BPF_B, value_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
        *insn++ = BPF_ALU32_IMM(BPF_AND, value_reg, SKB_TSTAMP_TYPE_MASK);
#ifdef __BIG_ENDIAN_BITFIELD
        *insn++ = BPF_ALU32_IMM(BPF_RSH, value_reg, SKB_TSTAMP_TYPE_RSHIFT);
#else
        BUILD_BUG_ON(!(SKB_TSTAMP_TYPE_MASK & 0x1));
#endif

        return insn;
}

static struct bpf_insn *bpf_convert_shinfo_access(__u8 dst_reg, __u8 skb_reg,
                                                  struct bpf_insn *insn)
{
        /* si->dst_reg = skb_shinfo(SKB); */
#ifdef NET_SKBUFF_DATA_USES_OFFSET
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
                              BPF_REG_AX, skb_reg,
                              offsetof(struct sk_buff, end));
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, head),
                              dst_reg, skb_reg,
                              offsetof(struct sk_buff, head));
        *insn++ = BPF_ALU64_REG(BPF_ADD, dst_reg, BPF_REG_AX);
#else
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, end),
                              dst_reg, skb_reg,
                              offsetof(struct sk_buff, end));
#endif

        return insn;
}

static struct bpf_insn *bpf_convert_tstamp_read(const struct bpf_prog *prog,
                                                const struct bpf_insn *si,
                                                struct bpf_insn *insn)
{
        __u8 value_reg = si->dst_reg;
        __u8 skb_reg = si->src_reg;

#ifdef CONFIG_NET_XGRESS
        /* If the tstamp_type is read,
         * the bpf prog is aware the tstamp could have delivery time.
         * Thus, read skb->tstamp as is if tstamp_type_access is true.
         */
        if (!prog->tstamp_type_access) {
                /* AX is needed because src_reg and dst_reg could be the same */
                __u8 tmp_reg = BPF_REG_AX;

                *insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
                /* check if ingress mask bits is set */
                *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
                *insn++ = BPF_JMP_A(4);
                *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, SKB_TSTAMP_TYPE_MASK, 1);
                *insn++ = BPF_JMP_A(2);
                /* skb->tc_at_ingress && skb->tstamp_type,
                 * read 0 as the (rcv) timestamp.
                 */
                *insn++ = BPF_MOV64_IMM(value_reg, 0);
                *insn++ = BPF_JMP_A(1);
        }
#endif

        *insn++ = BPF_LDX_MEM(BPF_DW, value_reg, skb_reg,
                              offsetof(struct sk_buff, tstamp));
        return insn;
}

static struct bpf_insn *bpf_convert_tstamp_write(const struct bpf_prog *prog,
                                                 const struct bpf_insn *si,
                                                 struct bpf_insn *insn)
{
        __u8 value_reg = si->src_reg;
        __u8 skb_reg = si->dst_reg;

#ifdef CONFIG_NET_XGRESS
        /* If the tstamp_type is read,
         * the bpf prog is aware the tstamp could have delivery time.
         * Thus, write skb->tstamp as is if tstamp_type_access is true.
         * Otherwise, writing at ingress will have to clear the
         * skb->tstamp_type bit also.
         */
        if (!prog->tstamp_type_access) {
                __u8 tmp_reg = BPF_REG_AX;

                *insn++ = BPF_LDX_MEM(BPF_B, tmp_reg, skb_reg, SKB_BF_MONO_TC_OFFSET);
                /* Writing __sk_buff->tstamp as ingress, goto <clear> */
                *insn++ = BPF_JMP32_IMM(BPF_JSET, tmp_reg, TC_AT_INGRESS_MASK, 1);
                /* goto <store> */
                *insn++ = BPF_JMP_A(2);
                /* <clear>: skb->tstamp_type */
                *insn++ = BPF_ALU32_IMM(BPF_AND, tmp_reg, ~SKB_TSTAMP_TYPE_MASK);
                *insn++ = BPF_STX_MEM(BPF_B, skb_reg, tmp_reg, SKB_BF_MONO_TC_OFFSET);
        }
#endif

        /* <store>: skb->tstamp = tstamp */
        *insn++ = BPF_RAW_INSN(BPF_CLASS(si->code) | BPF_DW | BPF_MEM,
                               skb_reg, value_reg, offsetof(struct sk_buff, tstamp), si->imm);
        return insn;
}

#define BPF_EMIT_STORE(size, si, off)                                        \
        BPF_RAW_INSN(BPF_CLASS((si)->code) | (size) | BPF_MEM,                \
                     (si)->dst_reg, (si)->src_reg, (off), (si)->imm)

static u32 bpf_convert_ctx_access(enum bpf_access_type type,
                                  const struct bpf_insn *si,
                                  struct bpf_insn *insn_buf,
                                  struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
        int off;

        switch (si->off) {
        case offsetof(struct __sk_buff, len):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, len, 4,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, protocol):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, protocol, 2,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, vlan_proto):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, vlan_proto, 2,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, priority):
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si,
                                                 bpf_target_off(struct sk_buff, priority, 4,
                                                                target_size));
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                              bpf_target_off(struct sk_buff, priority, 4,
                                                             target_size));
                break;

        case offsetof(struct __sk_buff, ingress_ifindex):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, skb_iif, 4,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, ifindex):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, dev));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct net_device, ifindex, 4,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, hash):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, hash, 4,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, mark):
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si,
                                                 bpf_target_off(struct sk_buff, mark, 4,
                                                                target_size));
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                              bpf_target_off(struct sk_buff, mark, 4,
                                                             target_size));
                break;

        case offsetof(struct __sk_buff, pkt_type):
                *target_size = 1;
                *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,
                                      PKT_TYPE_OFFSET);
                *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX);
#ifdef __BIG_ENDIAN_BITFIELD
                *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5);
#endif
                break;

        case offsetof(struct __sk_buff, queue_mapping):
                if (type == BPF_WRITE) {
                        u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);

                        if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
                                *insn++ = BPF_JMP_A(0); /* noop */
                                break;
                        }

                        if (BPF_CLASS(si->code) == BPF_STX)
                                *insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
                        *insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
                } else {
                        *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                              bpf_target_off(struct sk_buff,
                                                             queue_mapping,
                                                             2, target_size));
                }
                break;

        case offsetof(struct __sk_buff, vlan_present):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff,
                                                     vlan_all, 4, target_size));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_ALU32_IMM(BPF_MOV, si->dst_reg, 1);
                break;

        case offsetof(struct __sk_buff, vlan_tci):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, vlan_tci, 2,
                                                     target_size));
                break;

        case offsetof(struct __sk_buff, cb[0]) ...
             offsetofend(struct __sk_buff, cb[4]) - 1:
                BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, data) < 20);
                BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
                              offsetof(struct qdisc_skb_cb, data)) %
                             sizeof(__u64));

                prog->cb_access = 1;
                off  = si->off;
                off -= offsetof(struct __sk_buff, cb[0]);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct qdisc_skb_cb, data);
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
                else
                        *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
                                              si->src_reg, off);
                break;

        case offsetof(struct __sk_buff, tc_classid):
                BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, tc_classid) != 2);

                off  = si->off;
                off -= offsetof(struct __sk_buff, tc_classid);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct qdisc_skb_cb, tc_classid);
                *target_size = 2;
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_H, si, off);
                else
                        *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg,
                                              si->src_reg, off);
                break;

        case offsetof(struct __sk_buff, data):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, data));
                break;

        case offsetof(struct __sk_buff, data_meta):
                off  = si->off;
                off -= offsetof(struct __sk_buff, data_meta);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct bpf_skb_data_end, data_meta);
                *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
                                      si->src_reg, off);
                break;

        case offsetof(struct __sk_buff, data_end):
                off  = si->off;
                off -= offsetof(struct __sk_buff, data_end);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct bpf_skb_data_end, data_end);
                *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg,
                                      si->src_reg, off);
                break;

        case offsetof(struct __sk_buff, tc_index):
#ifdef CONFIG_NET_SCHED
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_H, si,
                                                 bpf_target_off(struct sk_buff, tc_index, 2,
                                                                target_size));
                else
                        *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                              bpf_target_off(struct sk_buff, tc_index, 2,
                                                             target_size));
#else
                *target_size = 2;
                if (type == BPF_WRITE)
                        *insn++ = BPF_MOV64_REG(si->dst_reg, si->dst_reg);
                else
                        *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct __sk_buff, napi_id):
#if defined(CONFIG_NET_RX_BUSY_POLL)
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct sk_buff, napi_id, 4,
                                                     target_size));
                *insn++ = BPF_JMP_IMM(BPF_JGE, si->dst_reg, MIN_NAPI_ID, 1);
                *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
#else
                *target_size = 4;
                *insn++ = BPF_MOV64_IMM(si->dst_reg, 0);
#endif
                break;
        case offsetof(struct __sk_buff, family):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct sock_common,
                                                     skc_family,
                                                     2, target_size));
                break;
        case offsetof(struct __sk_buff, remote_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct sock_common,
                                                     skc_daddr,
                                                     4, target_size));
                break;
        case offsetof(struct __sk_buff, local_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_rcv_saddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct sock_common,
                                                     skc_rcv_saddr,
                                                     4, target_size));
                break;
        case offsetof(struct __sk_buff, remote_ip6[0]) ...
             offsetof(struct __sk_buff, remote_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_daddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct __sk_buff, remote_ip6[0]);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_daddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;
        case offsetof(struct __sk_buff, local_ip6[0]) ...
             offsetof(struct __sk_buff, local_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_rcv_saddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct __sk_buff, local_ip6[0]);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_rcv_saddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct __sk_buff, remote_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct sock_common,
                                                     skc_dport,
                                                     2, target_size));
#ifndef __BIG_ENDIAN_BITFIELD
                *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
#endif
                break;

        case offsetof(struct __sk_buff, local_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct sock_common,
                                                     skc_num, 2, target_size));
                break;

        case offsetof(struct __sk_buff, tstamp):
                BUILD_BUG_ON(sizeof_field(struct sk_buff, tstamp) != 8);

                if (type == BPF_WRITE)
                        insn = bpf_convert_tstamp_write(prog, si, insn);
                else
                        insn = bpf_convert_tstamp_read(prog, si, insn);
                break;

        case offsetof(struct __sk_buff, tstamp_type):
                insn = bpf_convert_tstamp_type_read(si, insn);
                break;

        case offsetof(struct __sk_buff, gso_segs):
                insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_segs),
                                      si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct skb_shared_info,
                                                     gso_segs, 2,
                                                     target_size));
                break;
        case offsetof(struct __sk_buff, gso_size):
                insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct skb_shared_info, gso_size),
                                      si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct skb_shared_info,
                                                     gso_size, 2,
                                                     target_size));
                break;
        case offsetof(struct __sk_buff, wire_len):
                BUILD_BUG_ON(sizeof_field(struct qdisc_skb_cb, pkt_len) != 4);

                off = si->off;
                off -= offsetof(struct __sk_buff, wire_len);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct qdisc_skb_cb, pkt_len);
                *target_size = 4;
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);
                break;

        case offsetof(struct __sk_buff, sk):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, sk));
                break;
        case offsetof(struct __sk_buff, hwtstamp):
                BUILD_BUG_ON(sizeof_field(struct skb_shared_hwtstamps, hwtstamp) != 8);
                BUILD_BUG_ON(offsetof(struct skb_shared_hwtstamps, hwtstamp) != 0);

                insn = bpf_convert_shinfo_access(si->dst_reg, si->src_reg, insn);
                *insn++ = BPF_LDX_MEM(BPF_DW,
                                      si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct skb_shared_info,
                                                     hwtstamps, 8,
                                                     target_size));
                break;
        }

        return insn - insn_buf;
}

u32 bpf_sock_convert_ctx_access(enum bpf_access_type type,
                                const struct bpf_insn *si,
                                struct bpf_insn *insn_buf,
                                struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
        int off;

        switch (si->off) {
        case offsetof(struct bpf_sock, bound_dev_if):
                BUILD_BUG_ON(sizeof_field(struct sock, sk_bound_dev_if) != 4);

                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si,
                                                 offsetof(struct sock, sk_bound_dev_if));
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      offsetof(struct sock, sk_bound_dev_if));
                break;

        case offsetof(struct bpf_sock, mark):
                BUILD_BUG_ON(sizeof_field(struct sock, sk_mark) != 4);

                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si,
                                                 offsetof(struct sock, sk_mark));
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      offsetof(struct sock, sk_mark));
                break;

        case offsetof(struct bpf_sock, priority):
                BUILD_BUG_ON(sizeof_field(struct sock, sk_priority) != 4);

                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si,
                                                 offsetof(struct sock, sk_priority));
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      offsetof(struct sock, sk_priority));
                break;

        case offsetof(struct bpf_sock, family):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock_common, skc_family),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common,
                                       skc_family,
                                       sizeof_field(struct sock_common,
                                                    skc_family),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, type):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock, sk_type),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock, sk_type,
                                       sizeof_field(struct sock, sk_type),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, protocol):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock, sk_protocol),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock, sk_protocol,
                                       sizeof_field(struct sock, sk_protocol),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, src_ip4):
                *insn++ = BPF_LDX_MEM(
                        BPF_SIZE(si->code), si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common, skc_rcv_saddr,
                                       sizeof_field(struct sock_common,
                                                    skc_rcv_saddr),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, dst_ip4):
                *insn++ = BPF_LDX_MEM(
                        BPF_SIZE(si->code), si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common, skc_daddr,
                                       sizeof_field(struct sock_common,
                                                    skc_daddr),
                                       target_size));
                break;

        case bpf_ctx_range_till(struct bpf_sock, src_ip6[0], src_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                off = si->off;
                off -= offsetof(struct bpf_sock, src_ip6[0]);
                *insn++ = BPF_LDX_MEM(
                        BPF_SIZE(si->code), si->dst_reg, si->src_reg,
                        bpf_target_off(
                                struct sock_common,
                                skc_v6_rcv_saddr.s6_addr32[0],
                                sizeof_field(struct sock_common,
                                             skc_v6_rcv_saddr.s6_addr32[0]),
                                target_size) + off);
#else
                (void)off;
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case bpf_ctx_range_till(struct bpf_sock, dst_ip6[0], dst_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                off = si->off;
                off -= offsetof(struct bpf_sock, dst_ip6[0]);
                *insn++ = BPF_LDX_MEM(
                        BPF_SIZE(si->code), si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common,
                                       skc_v6_daddr.s6_addr32[0],
                                       sizeof_field(struct sock_common,
                                                    skc_v6_daddr.s6_addr32[0]),
                                       target_size) + off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
                *target_size = 4;
#endif
                break;

        case offsetof(struct bpf_sock, src_port):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock_common, skc_num),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common, skc_num,
                                       sizeof_field(struct sock_common,
                                                    skc_num),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, dst_port):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock_common, skc_dport),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common, skc_dport,
                                       sizeof_field(struct sock_common,
                                                    skc_dport),
                                       target_size));
                break;

        case offsetof(struct bpf_sock, state):
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock_common, skc_state),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock_common, skc_state,
                                       sizeof_field(struct sock_common,
                                                    skc_state),
                                       target_size));
                break;
        case offsetof(struct bpf_sock, rx_queue_mapping):
#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
                *insn++ = BPF_LDX_MEM(
                        BPF_FIELD_SIZEOF(struct sock, sk_rx_queue_mapping),
                        si->dst_reg, si->src_reg,
                        bpf_target_off(struct sock, sk_rx_queue_mapping,
                                       sizeof_field(struct sock,
                                                    sk_rx_queue_mapping),
                                       target_size));
                *insn++ = BPF_JMP_IMM(BPF_JNE, si->dst_reg, NO_QUEUE_MAPPING,
                                      1);
                *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
#else
                *insn++ = BPF_MOV64_IMM(si->dst_reg, -1);
                *target_size = 2;
#endif
                break;
        }

        return insn - insn_buf;
}

static u32 tc_cls_act_convert_ctx_access(enum bpf_access_type type,
                                         const struct bpf_insn *si,
                                         struct bpf_insn *insn_buf,
                                         struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct __sk_buff, ifindex):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, dev),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_buff, dev));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct net_device, ifindex, 4,
                                                     target_size));
                break;
        default:
                return bpf_convert_ctx_access(type, si, insn_buf, prog,
                                              target_size);
        }

        return insn - insn_buf;
}

static u32 xdp_convert_ctx_access(enum bpf_access_type type,
                                  const struct bpf_insn *si,
                                  struct bpf_insn *insn_buf,
                                  struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct xdp_md, data):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, data));
                break;
        case offsetof(struct xdp_md, data_meta):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_meta),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, data_meta));
                break;
        case offsetof(struct xdp_md, data_end):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, data_end),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, data_end));
                break;
        case offsetof(struct xdp_md, ingress_ifindex):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, rxq));
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_rxq_info, dev),
                                      si->dst_reg, si->dst_reg,
                                      offsetof(struct xdp_rxq_info, dev));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct net_device, ifindex));
                break;
        case offsetof(struct xdp_md, rx_queue_index):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, rxq),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, rxq));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct xdp_rxq_info,
                                               queue_index));
                break;
        case offsetof(struct xdp_md, egress_ifindex):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_buff, txq),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct xdp_buff, txq));
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_txq_info, dev),
                                      si->dst_reg, si->dst_reg,
                                      offsetof(struct xdp_txq_info, dev));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct net_device, ifindex));
                break;
        }

        return insn - insn_buf;
}

/* SOCK_ADDR_LOAD_NESTED_FIELD() loads Nested Field S.F.NF where S is type of
 * context Structure, F is Field in context structure that contains a pointer
 * to Nested Structure of type NS that has the field NF.
 *
 * SIZE encodes the load size (BPF_B, BPF_H, etc). It's up to caller to make
 * sure that SIZE is not greater than actual size of S.F.NF.
 *
 * If offset OFF is provided, the load happens from that offset relative to
 * offset of NF.
 */
#define SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF)               \
        do {                                                                       \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), si->dst_reg,     \
                                      si->src_reg, offsetof(S, F));               \
                *insn++ = BPF_LDX_MEM(                                               \
                        SIZE, si->dst_reg, si->dst_reg,                               \
                        bpf_target_off(NS, NF, sizeof_field(NS, NF),               \
                                       target_size)                               \
                                + OFF);                                               \
        } while (0)

#define SOCK_ADDR_LOAD_NESTED_FIELD(S, NS, F, NF)                               \
        SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(S, NS, F, NF,                       \
                                             BPF_FIELD_SIZEOF(NS, NF), 0)

/* SOCK_ADDR_STORE_NESTED_FIELD_OFF() has semantic similar to
 * SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF() but for store operation.
 *
 * In addition it uses Temporary Field TF (member of struct S) as the 3rd
 * "register" since two registers available in convert_ctx_access are not
 * enough: we can't override neither SRC, since it contains value to store, nor
 * DST since it contains pointer to context that may be used by later
 * instructions. But we need a temporary place to save pointer to nested
 * structure whose field we want to store to.
 */
#define SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE, OFF, TF)               \
        do {                                                                       \
                int tmp_reg = BPF_REG_9;                                       \
                if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg)               \
                        --tmp_reg;                                               \
                if (si->src_reg == tmp_reg || si->dst_reg == tmp_reg)               \
                        --tmp_reg;                                               \
                *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, tmp_reg,               \
                                      offsetof(S, TF));                               \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(S, F), tmp_reg,               \
                                      si->dst_reg, offsetof(S, F));               \
                *insn++ = BPF_RAW_INSN(SIZE | BPF_MEM | BPF_CLASS(si->code),   \
                                       tmp_reg, si->src_reg,                       \
                        bpf_target_off(NS, NF, sizeof_field(NS, NF),               \
                                       target_size)                               \
                                       + OFF,                                       \
                                       si->imm);                               \
                *insn++ = BPF_LDX_MEM(BPF_DW, tmp_reg, si->dst_reg,               \
                                      offsetof(S, TF));                               \
        } while (0)

#define SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(S, NS, F, NF, SIZE, OFF, \
                                                      TF)                       \
        do {                                                                       \
                if (type == BPF_WRITE) {                                       \
                        SOCK_ADDR_STORE_NESTED_FIELD_OFF(S, NS, F, NF, SIZE,   \
                                                         OFF, TF);               \
                } else {                                                       \
                        SOCK_ADDR_LOAD_NESTED_FIELD_SIZE_OFF(                       \
                                S, NS, F, NF, SIZE, OFF);  \
                }                                                               \
        } while (0)

static u32 sock_addr_convert_ctx_access(enum bpf_access_type type,
                                        const struct bpf_insn *si,
                                        struct bpf_insn *insn_buf,
                                        struct bpf_prog *prog, u32 *target_size)
{
        int off, port_size = sizeof_field(struct sockaddr_in6, sin6_port);
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct bpf_sock_addr, user_family):
                SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
                                            struct sockaddr, uaddr, sa_family);
                break;

        case offsetof(struct bpf_sock_addr, user_ip4):
                SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
                        struct bpf_sock_addr_kern, struct sockaddr_in, uaddr,
                        sin_addr, BPF_SIZE(si->code), 0, tmp_reg);
                break;

        case bpf_ctx_range_till(struct bpf_sock_addr, user_ip6[0], user_ip6[3]):
                off = si->off;
                off -= offsetof(struct bpf_sock_addr, user_ip6[0]);
                SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
                        struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
                        sin6_addr.s6_addr32[0], BPF_SIZE(si->code), off,
                        tmp_reg);
                break;

        case offsetof(struct bpf_sock_addr, user_port):
                /* To get port we need to know sa_family first and then treat
                 * sockaddr as either sockaddr_in or sockaddr_in6.
                 * Though we can simplify since port field has same offset and
                 * size in both structures.
                 * Here we check this invariant and use just one of the
                 * structures if it's true.
                 */
                BUILD_BUG_ON(offsetof(struct sockaddr_in, sin_port) !=
                             offsetof(struct sockaddr_in6, sin6_port));
                BUILD_BUG_ON(sizeof_field(struct sockaddr_in, sin_port) !=
                             sizeof_field(struct sockaddr_in6, sin6_port));
                /* Account for sin6_port being smaller than user_port. */
                port_size = min(port_size, BPF_LDST_BYTES(si));
                SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
                        struct bpf_sock_addr_kern, struct sockaddr_in6, uaddr,
                        sin6_port, bytes_to_bpf_size(port_size), 0, tmp_reg);
                break;

        case offsetof(struct bpf_sock_addr, family):
                SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
                                            struct sock, sk, sk_family);
                break;

        case offsetof(struct bpf_sock_addr, type):
                SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
                                            struct sock, sk, sk_type);
                break;

        case offsetof(struct bpf_sock_addr, protocol):
                SOCK_ADDR_LOAD_NESTED_FIELD(struct bpf_sock_addr_kern,
                                            struct sock, sk, sk_protocol);
                break;

        case offsetof(struct bpf_sock_addr, msg_src_ip4):
                /* Treat t_ctx as struct in_addr for msg_src_ip4. */
                SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
                        struct bpf_sock_addr_kern, struct in_addr, t_ctx,
                        s_addr, BPF_SIZE(si->code), 0, tmp_reg);
                break;

        case bpf_ctx_range_till(struct bpf_sock_addr, msg_src_ip6[0],
                                msg_src_ip6[3]):
                off = si->off;
                off -= offsetof(struct bpf_sock_addr, msg_src_ip6[0]);
                /* Treat t_ctx as struct in6_addr for msg_src_ip6. */
                SOCK_ADDR_LOAD_OR_STORE_NESTED_FIELD_SIZE_OFF(
                        struct bpf_sock_addr_kern, struct in6_addr, t_ctx,
                        s6_addr32[0], BPF_SIZE(si->code), off, tmp_reg);
                break;
        case offsetof(struct bpf_sock_addr, sk):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_addr_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_addr_kern, sk));
                break;
        }

        return insn - insn_buf;
}

static u32 sock_ops_convert_ctx_access(enum bpf_access_type type,
                                       const struct bpf_insn *si,
                                       struct bpf_insn *insn_buf,
                                       struct bpf_prog *prog,
                                       u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
        int off;

/* Helper macro for adding read access to tcp_sock or sock fields. */
#define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)                              \
        do {                                                                      \
                int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 2;     \
                BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) >                      \
                             sizeof_field(struct bpf_sock_ops, BPF_FIELD));   \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                if (si->dst_reg == si->src_reg) {                              \
                        *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg,              \
                                          offsetof(struct bpf_sock_ops_kern,  \
                                          temp));                              \
                        fullsock_reg = reg;                                      \
                        jmp += 2;                                              \
                }                                                              \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern,     \
                                                is_locked_tcp_sock),              \
                                      fullsock_reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                               is_locked_tcp_sock));              \
                *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp);              \
                if (si->dst_reg == si->src_reg)                                      \
                        *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                      temp));                                      \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern, sk),\
                                      si->dst_reg, si->src_reg,                      \
                                      offsetof(struct bpf_sock_ops_kern, sk));\
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ,                      \
                                                       OBJ_FIELD),              \
                                      si->dst_reg, si->dst_reg,                      \
                                      offsetof(OBJ, OBJ_FIELD));              \
                if (si->dst_reg == si->src_reg)        {                              \
                        *insn++ = BPF_JMP_A(1);                                      \
                        *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                      temp));                                      \
                }                                                              \
        } while (0)

#define SOCK_OPS_GET_SK()                                                              \
        do {                                                                      \
                int fullsock_reg = si->dst_reg, reg = BPF_REG_9, jmp = 1;     \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                if (si->dst_reg == si->src_reg) {                              \
                        *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg,              \
                                          offsetof(struct bpf_sock_ops_kern,  \
                                          temp));                              \
                        fullsock_reg = reg;                                      \
                        jmp += 2;                                              \
                }                                                              \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern,     \
                                                is_fullsock),                      \
                                      fullsock_reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                               is_fullsock));                      \
                *insn++ = BPF_JMP_IMM(BPF_JEQ, fullsock_reg, 0, jmp);              \
                if (si->dst_reg == si->src_reg)                                      \
                        *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                      temp));                                      \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern, sk),\
                                      si->dst_reg, si->src_reg,                      \
                                      offsetof(struct bpf_sock_ops_kern, sk));\
                if (si->dst_reg == si->src_reg)        {                              \
                        *insn++ = BPF_JMP_A(1);                                      \
                        *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg,              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                      temp));                                      \
                }                                                              \
        } while (0)

#define SOCK_OPS_GET_TCP_SOCK_FIELD(FIELD) \
                SOCK_OPS_GET_FIELD(FIELD, FIELD, struct tcp_sock)

/* Helper macro for adding write access to tcp_sock or sock fields.
 * The macro is called with two registers, dst_reg which contains a pointer
 * to ctx (context) and src_reg which contains the value that should be
 * stored. However, we need an additional register since we cannot overwrite
 * dst_reg because it may be used later in the program.
 * Instead we "borrow" one of the other register. We first save its value
 * into a new (temp) field in bpf_sock_ops_kern, use it, and then restore
 * it at the end of the macro.
 */
#define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)                              \
        do {                                                                      \
                int reg = BPF_REG_9;                                              \
                BUILD_BUG_ON(sizeof_field(OBJ, OBJ_FIELD) >                      \
                             sizeof_field(struct bpf_sock_ops, BPF_FIELD));   \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                if (si->dst_reg == reg || si->src_reg == reg)                      \
                        reg--;                                                      \
                *insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg,                      \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                               temp));                              \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern,     \
                                                is_locked_tcp_sock),              \
                                      reg, si->dst_reg,                              \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                               is_locked_tcp_sock));              \
                *insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2);                      \
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(                              \
                                                struct bpf_sock_ops_kern, sk),\
                                      reg, si->dst_reg,                              \
                                      offsetof(struct bpf_sock_ops_kern, sk));\
                *insn++ = BPF_RAW_INSN(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD) |     \
                                       BPF_MEM | BPF_CLASS(si->code),              \
                                       reg, si->src_reg,                      \
                                       offsetof(OBJ, OBJ_FIELD),              \
                                       si->imm);                              \
                *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg,                      \
                                      offsetof(struct bpf_sock_ops_kern,      \
                                               temp));                              \
        } while (0)

#define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE)              \
        do {                                                                      \
                if (TYPE == BPF_WRITE)                                              \
                        SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);              \
                else                                                              \
                        SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);              \
        } while (0)

        switch (si->off) {
        case offsetof(struct bpf_sock_ops, op):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       op),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, op));
                break;

        case offsetof(struct bpf_sock_ops, replylong[0]) ...
             offsetof(struct bpf_sock_ops, replylong[3]):
                BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, reply) !=
                             sizeof_field(struct bpf_sock_ops_kern, reply));
                BUILD_BUG_ON(sizeof_field(struct bpf_sock_ops, replylong) !=
                             sizeof_field(struct bpf_sock_ops_kern, replylong));
                off = si->off;
                off -= offsetof(struct bpf_sock_ops, replylong[0]);
                off += offsetof(struct bpf_sock_ops_kern, replylong[0]);
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_W, si, off);
                else
                        *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                              off);
                break;

        case offsetof(struct bpf_sock_ops, family):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                              struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_family));
                break;

        case offsetof(struct bpf_sock_ops, remote_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_daddr));
                break;

        case offsetof(struct bpf_sock_ops, local_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_rcv_saddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                              struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_rcv_saddr));
                break;

        case offsetof(struct bpf_sock_ops, remote_ip6[0]) ...
             offsetof(struct bpf_sock_ops, remote_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_daddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct bpf_sock_ops, remote_ip6[0]);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_daddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct bpf_sock_ops, local_ip6[0]) ...
             offsetof(struct bpf_sock_ops, local_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_rcv_saddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct bpf_sock_ops, local_ip6[0]);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_rcv_saddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct bpf_sock_ops, remote_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_dport));
#ifndef __BIG_ENDIAN_BITFIELD
                *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
#endif
                break;

        case offsetof(struct bpf_sock_ops, local_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_num));
                break;

        case offsetof(struct bpf_sock_ops, is_fullsock):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern,
                                                is_fullsock),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               is_fullsock));
                break;

        case offsetof(struct bpf_sock_ops, state):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_state) != 1);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_state));
                break;

        case offsetof(struct bpf_sock_ops, rtt_min):
                BUILD_BUG_ON(sizeof_field(struct tcp_sock, rtt_min) !=
                             sizeof(struct minmax));
                BUILD_BUG_ON(sizeof(struct minmax) <
                             sizeof(struct minmax_sample));

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct bpf_sock_ops_kern, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct tcp_sock, rtt_min) +
                                      sizeof_field(struct minmax_sample, t));
                break;

        case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):
                SOCK_OPS_GET_FIELD(bpf_sock_ops_cb_flags, bpf_sock_ops_cb_flags,
                                   struct tcp_sock);
                break;

        case offsetof(struct bpf_sock_ops, sk_txhash):
                SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
                                          struct sock, type);
                break;
        case offsetof(struct bpf_sock_ops, snd_cwnd):
                SOCK_OPS_GET_TCP_SOCK_FIELD(snd_cwnd);
                break;
        case offsetof(struct bpf_sock_ops, srtt_us):
                SOCK_OPS_GET_TCP_SOCK_FIELD(srtt_us);
                break;
        case offsetof(struct bpf_sock_ops, snd_ssthresh):
                SOCK_OPS_GET_TCP_SOCK_FIELD(snd_ssthresh);
                break;
        case offsetof(struct bpf_sock_ops, rcv_nxt):
                SOCK_OPS_GET_TCP_SOCK_FIELD(rcv_nxt);
                break;
        case offsetof(struct bpf_sock_ops, snd_nxt):
                SOCK_OPS_GET_TCP_SOCK_FIELD(snd_nxt);
                break;
        case offsetof(struct bpf_sock_ops, snd_una):
                SOCK_OPS_GET_TCP_SOCK_FIELD(snd_una);
                break;
        case offsetof(struct bpf_sock_ops, mss_cache):
                SOCK_OPS_GET_TCP_SOCK_FIELD(mss_cache);
                break;
        case offsetof(struct bpf_sock_ops, ecn_flags):
                SOCK_OPS_GET_TCP_SOCK_FIELD(ecn_flags);
                break;
        case offsetof(struct bpf_sock_ops, rate_delivered):
                SOCK_OPS_GET_TCP_SOCK_FIELD(rate_delivered);
                break;
        case offsetof(struct bpf_sock_ops, rate_interval_us):
                SOCK_OPS_GET_TCP_SOCK_FIELD(rate_interval_us);
                break;
        case offsetof(struct bpf_sock_ops, packets_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(packets_out);
                break;
        case offsetof(struct bpf_sock_ops, retrans_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(retrans_out);
                break;
        case offsetof(struct bpf_sock_ops, total_retrans):
                SOCK_OPS_GET_TCP_SOCK_FIELD(total_retrans);
                break;
        case offsetof(struct bpf_sock_ops, segs_in):
                SOCK_OPS_GET_TCP_SOCK_FIELD(segs_in);
                break;
        case offsetof(struct bpf_sock_ops, data_segs_in):
                SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_in);
                break;
        case offsetof(struct bpf_sock_ops, segs_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(segs_out);
                break;
        case offsetof(struct bpf_sock_ops, data_segs_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(data_segs_out);
                break;
        case offsetof(struct bpf_sock_ops, lost_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(lost_out);
                break;
        case offsetof(struct bpf_sock_ops, sacked_out):
                SOCK_OPS_GET_TCP_SOCK_FIELD(sacked_out);
                break;
        case offsetof(struct bpf_sock_ops, bytes_received):
                SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_received);
                break;
        case offsetof(struct bpf_sock_ops, bytes_acked):
                SOCK_OPS_GET_TCP_SOCK_FIELD(bytes_acked);
                break;
        case offsetof(struct bpf_sock_ops, sk):
                SOCK_OPS_GET_SK();
                break;
        case offsetof(struct bpf_sock_ops, skb_data_end):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       skb_data_end),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               skb_data_end));
                break;
        case offsetof(struct bpf_sock_ops, skb_data):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       skb),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               skb));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
                                      si->dst_reg, si->dst_reg,
                                      offsetof(struct sk_buff, data));
                break;
        case offsetof(struct bpf_sock_ops, skb_len):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       skb),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               skb));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
                                      si->dst_reg, si->dst_reg,
                                      offsetof(struct sk_buff, len));
                break;
        case offsetof(struct bpf_sock_ops, skb_tcp_flags):
                off = offsetof(struct sk_buff, cb);
                off += offsetof(struct tcp_skb_cb, tcp_flags);
                *target_size = sizeof_field(struct tcp_skb_cb, tcp_flags);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       skb),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               skb));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct tcp_skb_cb,
                                                       tcp_flags),
                                      si->dst_reg, si->dst_reg, off);
                break;
        case offsetof(struct bpf_sock_ops, skb_hwtstamp): {
                struct bpf_insn *jmp_on_null_skb;

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_ops_kern,
                                                       skb),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sock_ops_kern,
                                               skb));
                /* Reserve one insn to test skb == NULL */
                jmp_on_null_skb = insn++;
                insn = bpf_convert_shinfo_access(si->dst_reg, si->dst_reg, insn);
                *insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
                                      bpf_target_off(struct skb_shared_info,
                                                     hwtstamps, 8,
                                                     target_size));
                *jmp_on_null_skb = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0,
                                               insn - jmp_on_null_skb - 1);
                break;
        }
        }
        return insn - insn_buf;
}

/* data_end = skb->data + skb_headlen() */
static struct bpf_insn *bpf_convert_data_end_access(const struct bpf_insn *si,
                                                    struct bpf_insn *insn)
{
        int reg;
        int temp_reg_off = offsetof(struct sk_buff, cb) +
                           offsetof(struct sk_skb_cb, temp_reg);

        if (si->src_reg == si->dst_reg) {
                /* We need an extra register, choose and save a register. */
                reg = BPF_REG_9;
                if (si->src_reg == reg || si->dst_reg == reg)
                        reg--;
                if (si->src_reg == reg || si->dst_reg == reg)
                        reg--;
                *insn++ = BPF_STX_MEM(BPF_DW, si->src_reg, reg, temp_reg_off);
        } else {
                reg = si->dst_reg;
        }

        /* reg = skb->data */
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data),
                              reg, si->src_reg,
                              offsetof(struct sk_buff, data));
        /* AX = skb->len */
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, len),
                              BPF_REG_AX, si->src_reg,
                              offsetof(struct sk_buff, len));
        /* reg = skb->data + skb->len */
        *insn++ = BPF_ALU64_REG(BPF_ADD, reg, BPF_REG_AX);
        /* AX = skb->data_len */
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_buff, data_len),
                              BPF_REG_AX, si->src_reg,
                              offsetof(struct sk_buff, data_len));

        /* reg = skb->data + skb->len - skb->data_len */
        *insn++ = BPF_ALU64_REG(BPF_SUB, reg, BPF_REG_AX);

        if (si->src_reg == si->dst_reg) {
                /* Restore the saved register */
                *insn++ = BPF_MOV64_REG(BPF_REG_AX, si->src_reg);
                *insn++ = BPF_MOV64_REG(si->dst_reg, reg);
                *insn++ = BPF_LDX_MEM(BPF_DW, reg, BPF_REG_AX, temp_reg_off);
        }

        return insn;
}

static u32 sk_skb_convert_ctx_access(enum bpf_access_type type,
                                     const struct bpf_insn *si,
                                     struct bpf_insn *insn_buf,
                                     struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
        int off;

        switch (si->off) {
        case offsetof(struct __sk_buff, data_end):
                insn = bpf_convert_data_end_access(si, insn);
                break;
        case offsetof(struct __sk_buff, cb[0]) ...
             offsetofend(struct __sk_buff, cb[4]) - 1:
                BUILD_BUG_ON(sizeof_field(struct sk_skb_cb, data) < 20);
                BUILD_BUG_ON((offsetof(struct sk_buff, cb) +
                              offsetof(struct sk_skb_cb, data)) %
                             sizeof(__u64));

                prog->cb_access = 1;
                off  = si->off;
                off -= offsetof(struct __sk_buff, cb[0]);
                off += offsetof(struct sk_buff, cb);
                off += offsetof(struct sk_skb_cb, data);
                if (type == BPF_WRITE)
                        *insn++ = BPF_EMIT_STORE(BPF_SIZE(si->code), si, off);
                else
                        *insn++ = BPF_LDX_MEM(BPF_SIZE(si->code), si->dst_reg,
                                              si->src_reg, off);
                break;


        default:
                return bpf_convert_ctx_access(type, si, insn_buf, prog,
                                              target_size);
        }

        return insn - insn_buf;
}

static u32 sk_msg_convert_ctx_access(enum bpf_access_type type,
                                     const struct bpf_insn *si,
                                     struct bpf_insn *insn_buf,
                                     struct bpf_prog *prog, u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;
#if IS_ENABLED(CONFIG_IPV6)
        int off;
#endif

        /* convert ctx uses the fact sg element is first in struct */
        BUILD_BUG_ON(offsetof(struct sk_msg, sg) != 0);

        switch (si->off) {
        case offsetof(struct sk_msg_md, data):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, data));
                break;
        case offsetof(struct sk_msg_md, data_end):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, data_end),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, data_end));
                break;
        case offsetof(struct sk_msg_md, family):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_family) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                              struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_family));
                break;

        case offsetof(struct sk_msg_md, remote_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_daddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_daddr));
                break;

        case offsetof(struct sk_msg_md, local_ip4):
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_rcv_saddr) != 4);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                              struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_rcv_saddr));
                break;

        case offsetof(struct sk_msg_md, remote_ip6[0]) ...
             offsetof(struct sk_msg_md, remote_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_daddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct sk_msg_md, remote_ip6[0]);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_daddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct sk_msg_md, local_ip6[0]) ...
             offsetof(struct sk_msg_md, local_ip6[3]):
#if IS_ENABLED(CONFIG_IPV6)
                BUILD_BUG_ON(sizeof_field(struct sock_common,
                                          skc_v6_rcv_saddr.s6_addr32[0]) != 4);

                off = si->off;
                off -= offsetof(struct sk_msg_md, local_ip6[0]);
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common,
                                               skc_v6_rcv_saddr.s6_addr32[0]) +
                                      off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;

        case offsetof(struct sk_msg_md, remote_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_dport) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_dport));
#ifndef __BIG_ENDIAN_BITFIELD
                *insn++ = BPF_ALU32_IMM(BPF_LSH, si->dst_reg, 16);
#endif
                break;

        case offsetof(struct sk_msg_md, local_port):
                BUILD_BUG_ON(sizeof_field(struct sock_common, skc_num) != 2);

                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
                                                struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->dst_reg,
                                      offsetof(struct sock_common, skc_num));
                break;

        case offsetof(struct sk_msg_md, size):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_sg, size),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg_sg, size));
                break;

        case offsetof(struct sk_msg_md, sk):
                *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg, sk),
                                      si->dst_reg, si->src_reg,
                                      offsetof(struct sk_msg, sk));
                break;
        }

        return insn - insn_buf;
}

const struct bpf_verifier_ops sk_filter_verifier_ops = {
        .get_func_proto                = sk_filter_func_proto,
        .is_valid_access        = sk_filter_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
        .gen_ld_abs                = bpf_gen_ld_abs,
};

const struct bpf_prog_ops sk_filter_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
        .get_func_proto                = tc_cls_act_func_proto,
        .is_valid_access        = tc_cls_act_is_valid_access,
        .convert_ctx_access        = tc_cls_act_convert_ctx_access,
        .gen_prologue                = tc_cls_act_prologue,
        .gen_ld_abs                = bpf_gen_ld_abs,
        .btf_struct_access        = tc_cls_act_btf_struct_access,
};

const struct bpf_prog_ops tc_cls_act_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops xdp_verifier_ops = {
        .get_func_proto                = xdp_func_proto,
        .is_valid_access        = xdp_is_valid_access,
        .convert_ctx_access        = xdp_convert_ctx_access,
        .gen_prologue                = bpf_noop_prologue,
        .btf_struct_access        = xdp_btf_struct_access,
};

const struct bpf_prog_ops xdp_prog_ops = {
        .test_run                = bpf_prog_test_run_xdp,
};

const struct bpf_verifier_ops cg_skb_verifier_ops = {
        .get_func_proto                = cg_skb_func_proto,
        .is_valid_access        = cg_skb_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
};

const struct bpf_prog_ops cg_skb_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops lwt_in_verifier_ops = {
        .get_func_proto                = lwt_in_func_proto,
        .is_valid_access        = lwt_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
};

const struct bpf_prog_ops lwt_in_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops lwt_out_verifier_ops = {
        .get_func_proto                = lwt_out_func_proto,
        .is_valid_access        = lwt_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
};

const struct bpf_prog_ops lwt_out_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops lwt_xmit_verifier_ops = {
        .get_func_proto                = lwt_xmit_func_proto,
        .is_valid_access        = lwt_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
        .gen_prologue                = tc_cls_act_prologue,
};

const struct bpf_prog_ops lwt_xmit_prog_ops = {
        .test_run                = bpf_prog_test_run_skb,
};

const struct bpf_verifier_ops lwt_seg6local_verifier_ops = {
        .get_func_proto                = lwt_seg6local_func_proto,
        .is_valid_access        = lwt_is_valid_access,
        .convert_ctx_access        = bpf_convert_ctx_access,
};

const struct bpf_prog_ops lwt_seg6local_prog_ops = {
};

const struct bpf_verifier_ops cg_sock_verifier_ops = {
        .get_func_proto                = sock_filter_func_proto,
        .is_valid_access        = sock_filter_is_valid_access,
        .convert_ctx_access        = bpf_sock_convert_ctx_access,
};

const struct bpf_prog_ops cg_sock_prog_ops = {
};

const struct bpf_verifier_ops cg_sock_addr_verifier_ops = {
        .get_func_proto                = sock_addr_func_proto,
        .is_valid_access        = sock_addr_is_valid_access,
        .convert_ctx_access        = sock_addr_convert_ctx_access,
};

const struct bpf_prog_ops cg_sock_addr_prog_ops = {
};

const struct bpf_verifier_ops sock_ops_verifier_ops = {
        .get_func_proto                = sock_ops_func_proto,
        .is_valid_access        = sock_ops_is_valid_access,
        .convert_ctx_access        = sock_ops_convert_ctx_access,
};

const struct bpf_prog_ops sock_ops_prog_ops = {
};

const struct bpf_verifier_ops sk_skb_verifier_ops = {
        .get_func_proto                = sk_skb_func_proto,
        .is_valid_access        = sk_skb_is_valid_access,
        .convert_ctx_access        = sk_skb_convert_ctx_access,
        .gen_prologue                = sk_skb_prologue,
};

const struct bpf_prog_ops sk_skb_prog_ops = {
};

const struct bpf_verifier_ops sk_msg_verifier_ops = {
        .get_func_proto                = sk_msg_func_proto,
        .is_valid_access        = sk_msg_is_valid_access,
        .convert_ctx_access        = sk_msg_convert_ctx_access,
        .gen_prologue                = bpf_noop_prologue,
};

const struct bpf_prog_ops sk_msg_prog_ops = {
};

const struct bpf_verifier_ops flow_dissector_verifier_ops = {
        .get_func_proto                = flow_dissector_func_proto,
        .is_valid_access        = flow_dissector_is_valid_access,
        .convert_ctx_access        = flow_dissector_convert_ctx_access,
};

const struct bpf_prog_ops flow_dissector_prog_ops = {
        .test_run                = bpf_prog_test_run_flow_dissector,
};

int sk_detach_filter(struct sock *sk)
{
        int ret = -ENOENT;
        struct sk_filter *filter;

        if (sock_flag(sk, SOCK_FILTER_LOCKED))
                return -EPERM;

        filter = rcu_dereference_protected(sk->sk_filter,
                                           lockdep_sock_is_held(sk));
        if (filter) {
                RCU_INIT_POINTER(sk->sk_filter, NULL);
                sk_filter_uncharge(sk, filter);
                ret = 0;
        }

        return ret;
}
EXPORT_SYMBOL_GPL(sk_detach_filter);

int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len)
{
        struct sock_fprog_kern *fprog;
        struct sk_filter *filter;
        int ret = 0;

        sockopt_lock_sock(sk);
        filter = rcu_dereference_protected(sk->sk_filter,
                                           lockdep_sock_is_held(sk));
        if (!filter)
                goto out;

        /* We're copying the filter that has been originally attached,
         * so no conversion/decode needed anymore. eBPF programs that
         * have no original program cannot be dumped through this.
         */
        ret = -EACCES;
        fprog = filter->prog->orig_prog;
        if (!fprog)
                goto out;

        ret = fprog->len;
        if (!len)
                /* User space only enquires number of filter blocks. */
                goto out;

        ret = -EINVAL;
        if (len < fprog->len)
                goto out;

        ret = -EFAULT;
        if (copy_to_sockptr(optval, fprog->filter, bpf_classic_proglen(fprog)))
                goto out;

        /* Instead of bytes, the API requests to return the number
         * of filter blocks.
         */
        ret = fprog->len;
out:
        sockopt_release_sock(sk);
        return ret;
}

#ifdef CONFIG_INET
static void bpf_init_reuseport_kern(struct sk_reuseport_kern *reuse_kern,
                                    struct sock_reuseport *reuse,
                                    struct sock *sk, struct sk_buff *skb,
                                    struct sock *migrating_sk,
                                    u32 hash)
{
        reuse_kern->skb = skb;
        reuse_kern->sk = sk;
        reuse_kern->selected_sk = NULL;
        reuse_kern->migrating_sk = migrating_sk;
        reuse_kern->data_end = skb->data + skb_headlen(skb);
        reuse_kern->hash = hash;
        reuse_kern->reuseport_id = reuse->reuseport_id;
        reuse_kern->bind_inany = reuse->bind_inany;
}

struct sock *bpf_run_sk_reuseport(struct sock_reuseport *reuse, struct sock *sk,
                                  struct bpf_prog *prog, struct sk_buff *skb,
                                  struct sock *migrating_sk,
                                  u32 hash)
{
        struct sk_reuseport_kern reuse_kern;
        enum sk_action action;

        bpf_init_reuseport_kern(&reuse_kern, reuse, sk, skb, migrating_sk, hash);
        action = bpf_prog_run(prog, &reuse_kern);

        if (action == SK_PASS)
                return reuse_kern.selected_sk;
        else
                return ERR_PTR(-ECONNREFUSED);
}

BPF_CALL_4(sk_select_reuseport, struct sk_reuseport_kern *, reuse_kern,
           struct bpf_map *, map, void *, key, u32, flags)
{
        bool is_sockarray = map->map_type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
        struct sock_reuseport *reuse;
        struct sock *selected_sk;
        int err;

        selected_sk = map->ops->map_lookup_elem(map, key);
        if (!selected_sk)
                return -ENOENT;

        reuse = rcu_dereference(selected_sk->sk_reuseport_cb);
        if (!reuse) {
                /* reuseport_array has only sk with non NULL sk_reuseport_cb.
                 * The only (!reuse) case here is - the sk has already been
                 * unhashed (e.g. by close()), so treat it as -ENOENT.
                 *
                 * Other maps (e.g. sock_map) do not provide this guarantee and
                 * the sk may never be in the reuseport group to begin with.
                 */
                err = is_sockarray ? -ENOENT : -EINVAL;
                goto error;
        }

        if (unlikely(reuse->reuseport_id != reuse_kern->reuseport_id)) {
                struct sock *sk = reuse_kern->sk;

                if (sk->sk_protocol != selected_sk->sk_protocol) {
                        err = -EPROTOTYPE;
                } else if (sk->sk_family != selected_sk->sk_family) {
                        err = -EAFNOSUPPORT;
                } else {
                        /* Catch all. Likely bound to a different sockaddr. */
                        err = -EBADFD;
                }
                goto error;
        }

        reuse_kern->selected_sk = selected_sk;

        return 0;
error:
        /* Lookup in sock_map can return TCP ESTABLISHED sockets. */
        if (sk_is_refcounted(selected_sk))
                sock_put(selected_sk);

        return err;
}

static const struct bpf_func_proto sk_select_reuseport_proto = {
        .func           = sk_select_reuseport,
        .gpl_only       = false,
        .ret_type       = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type      = ARG_CONST_MAP_PTR,
        .arg3_type      = ARG_PTR_TO_MAP_KEY,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(sk_reuseport_load_bytes,
           const struct sk_reuseport_kern *, reuse_kern, u32, offset,
           void *, to, u32, len)
{
        return ____bpf_skb_load_bytes(reuse_kern->skb, offset, to, len);
}

static const struct bpf_func_proto sk_reuseport_load_bytes_proto = {
        .func                = sk_reuseport_load_bytes,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
};

BPF_CALL_5(sk_reuseport_load_bytes_relative,
           const struct sk_reuseport_kern *, reuse_kern, u32, offset,
           void *, to, u32, len, u32, start_header)
{
        return ____bpf_skb_load_bytes_relative(reuse_kern->skb, offset, to,
                                               len, start_header);
}

static const struct bpf_func_proto sk_reuseport_load_bytes_relative_proto = {
        .func                = sk_reuseport_load_bytes_relative,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_ANYTHING,
        .arg3_type        = ARG_PTR_TO_UNINIT_MEM,
        .arg4_type        = ARG_CONST_SIZE,
        .arg5_type        = ARG_ANYTHING,
};

static const struct bpf_func_proto *
sk_reuseport_func_proto(enum bpf_func_id func_id,
                        const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_sk_select_reuseport:
                return &sk_select_reuseport_proto;
        case BPF_FUNC_skb_load_bytes:
                return &sk_reuseport_load_bytes_proto;
        case BPF_FUNC_skb_load_bytes_relative:
                return &sk_reuseport_load_bytes_relative_proto;
        case BPF_FUNC_get_socket_cookie:
                return &bpf_get_socket_ptr_cookie_proto;
        case BPF_FUNC_ktime_get_coarse_ns:
                return &bpf_ktime_get_coarse_ns_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }
}

static bool
sk_reuseport_is_valid_access(int off, int size,
                             enum bpf_access_type type,
                             const struct bpf_prog *prog,
                             struct bpf_insn_access_aux *info)
{
        const u32 size_default = sizeof(__u32);

        if (off < 0 || off >= sizeof(struct sk_reuseport_md) ||
            off % size || type != BPF_READ)
                return false;

        switch (off) {
        case offsetof(struct sk_reuseport_md, data):
                info->reg_type = PTR_TO_PACKET;
                return size == sizeof(__u64);

        case offsetof(struct sk_reuseport_md, data_end):
                info->reg_type = PTR_TO_PACKET_END;
                return size == sizeof(__u64);

        case offsetof(struct sk_reuseport_md, hash):
                return size == size_default;

        case offsetof(struct sk_reuseport_md, sk):
                info->reg_type = PTR_TO_SOCKET;
                return size == sizeof(__u64);

        case offsetof(struct sk_reuseport_md, migrating_sk):
                info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
                return size == sizeof(__u64);

        /* Fields that allow narrowing */
        case bpf_ctx_range(struct sk_reuseport_md, eth_protocol):
                if (size < sizeof_field(struct sk_buff, protocol))
                        return false;
                fallthrough;
        case bpf_ctx_range(struct sk_reuseport_md, ip_protocol):
        case bpf_ctx_range(struct sk_reuseport_md, bind_inany):
        case bpf_ctx_range(struct sk_reuseport_md, len):
                bpf_ctx_record_field_size(info, size_default);
                return bpf_ctx_narrow_access_ok(off, size, size_default);

        default:
                return false;
        }
}

#define SK_REUSEPORT_LOAD_FIELD(F) ({                                        \
        *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_reuseport_kern, F), \
                              si->dst_reg, si->src_reg,                        \
                              bpf_target_off(struct sk_reuseport_kern, F, \
                                             sizeof_field(struct sk_reuseport_kern, F), \
                                             target_size));                \
        })

#define SK_REUSEPORT_LOAD_SKB_FIELD(SKB_FIELD)                                \
        SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern,                \
                                    struct sk_buff,                        \
                                    skb,                                \
                                    SKB_FIELD)

#define SK_REUSEPORT_LOAD_SK_FIELD(SK_FIELD)                                \
        SOCK_ADDR_LOAD_NESTED_FIELD(struct sk_reuseport_kern,                \
                                    struct sock,                        \
                                    sk,                                        \
                                    SK_FIELD)

static u32 sk_reuseport_convert_ctx_access(enum bpf_access_type type,
                                           const struct bpf_insn *si,
                                           struct bpf_insn *insn_buf,
                                           struct bpf_prog *prog,
                                           u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct sk_reuseport_md, data):
                SK_REUSEPORT_LOAD_SKB_FIELD(data);
                break;

        case offsetof(struct sk_reuseport_md, len):
                SK_REUSEPORT_LOAD_SKB_FIELD(len);
                break;

        case offsetof(struct sk_reuseport_md, eth_protocol):
                SK_REUSEPORT_LOAD_SKB_FIELD(protocol);
                break;

        case offsetof(struct sk_reuseport_md, ip_protocol):
                SK_REUSEPORT_LOAD_SK_FIELD(sk_protocol);
                break;

        case offsetof(struct sk_reuseport_md, data_end):
                SK_REUSEPORT_LOAD_FIELD(data_end);
                break;

        case offsetof(struct sk_reuseport_md, hash):
                SK_REUSEPORT_LOAD_FIELD(hash);
                break;

        case offsetof(struct sk_reuseport_md, bind_inany):
                SK_REUSEPORT_LOAD_FIELD(bind_inany);
                break;

        case offsetof(struct sk_reuseport_md, sk):
                SK_REUSEPORT_LOAD_FIELD(sk);
                break;

        case offsetof(struct sk_reuseport_md, migrating_sk):
                SK_REUSEPORT_LOAD_FIELD(migrating_sk);
                break;
        }

        return insn - insn_buf;
}

const struct bpf_verifier_ops sk_reuseport_verifier_ops = {
        .get_func_proto                = sk_reuseport_func_proto,
        .is_valid_access        = sk_reuseport_is_valid_access,
        .convert_ctx_access        = sk_reuseport_convert_ctx_access,
};

const struct bpf_prog_ops sk_reuseport_prog_ops = {
};

DEFINE_STATIC_KEY_FALSE(bpf_sk_lookup_enabled);
EXPORT_SYMBOL(bpf_sk_lookup_enabled);

BPF_CALL_3(bpf_sk_lookup_assign, struct bpf_sk_lookup_kern *, ctx,
           struct sock *, sk, u64, flags)
{
        if (unlikely(flags & ~(BPF_SK_LOOKUP_F_REPLACE |
                               BPF_SK_LOOKUP_F_NO_REUSEPORT)))
                return -EINVAL;
        if (unlikely(sk && sk_is_refcounted(sk)))
                return -ESOCKTNOSUPPORT; /* reject non-RCU freed sockets */
        if (unlikely(sk && sk_is_tcp(sk) && sk->sk_state != TCP_LISTEN))
                return -ESOCKTNOSUPPORT; /* only accept TCP socket in LISTEN */
        if (unlikely(sk && sk_is_udp(sk) && sk->sk_state != TCP_CLOSE))
                return -ESOCKTNOSUPPORT; /* only accept UDP socket in CLOSE */

        /* Check if socket is suitable for packet L3/L4 protocol */
        if (sk && sk->sk_protocol != ctx->protocol)
                return -EPROTOTYPE;
        if (sk && sk->sk_family != ctx->family &&
            (sk->sk_family == AF_INET || ipv6_only_sock(sk)))
                return -EAFNOSUPPORT;

        if (ctx->selected_sk && !(flags & BPF_SK_LOOKUP_F_REPLACE))
                return -EEXIST;

        /* Select socket as lookup result */
        ctx->selected_sk = sk;
        ctx->no_reuseport = flags & BPF_SK_LOOKUP_F_NO_REUSEPORT;
        return 0;
}

static const struct bpf_func_proto bpf_sk_lookup_assign_proto = {
        .func                = bpf_sk_lookup_assign,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_PTR_TO_CTX,
        .arg2_type        = ARG_PTR_TO_SOCKET_OR_NULL,
        .arg3_type        = ARG_ANYTHING,
};

static const struct bpf_func_proto *
sk_lookup_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        switch (func_id) {
        case BPF_FUNC_perf_event_output:
                return &bpf_event_output_data_proto;
        case BPF_FUNC_sk_assign:
                return &bpf_sk_lookup_assign_proto;
        case BPF_FUNC_sk_release:
                return &bpf_sk_release_proto;
        default:
                return bpf_sk_base_func_proto(func_id, prog);
        }
}

static bool sk_lookup_is_valid_access(int off, int size,
                                      enum bpf_access_type type,
                                      const struct bpf_prog *prog,
                                      struct bpf_insn_access_aux *info)
{
        if (off < 0 || off >= sizeof(struct bpf_sk_lookup))
                return false;
        if (off % size != 0)
                return false;
        if (type != BPF_READ)
                return false;

        switch (off) {
        case offsetof(struct bpf_sk_lookup, sk):
                info->reg_type = PTR_TO_SOCKET_OR_NULL;
                return size == sizeof(__u64);

        case bpf_ctx_range(struct bpf_sk_lookup, family):
        case bpf_ctx_range(struct bpf_sk_lookup, protocol):
        case bpf_ctx_range(struct bpf_sk_lookup, remote_ip4):
        case bpf_ctx_range(struct bpf_sk_lookup, local_ip4):
        case bpf_ctx_range_till(struct bpf_sk_lookup, remote_ip6[0], remote_ip6[3]):
        case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]):
        case bpf_ctx_range(struct bpf_sk_lookup, local_port):
        case bpf_ctx_range(struct bpf_sk_lookup, ingress_ifindex):
                bpf_ctx_record_field_size(info, sizeof(__u32));
                return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32));

        case bpf_ctx_range(struct bpf_sk_lookup, remote_port):
                /* Allow 4-byte access to 2-byte field for backward compatibility */
                if (size == sizeof(__u32))
                        return true;
                bpf_ctx_record_field_size(info, sizeof(__be16));
                return bpf_ctx_narrow_access_ok(off, size, sizeof(__be16));

        case offsetofend(struct bpf_sk_lookup, remote_port) ...
             offsetof(struct bpf_sk_lookup, local_ip4) - 1:
                /* Allow access to zero padding for backward compatibility */
                bpf_ctx_record_field_size(info, sizeof(__u16));
                return bpf_ctx_narrow_access_ok(off, size, sizeof(__u16));

        default:
                return false;
        }
}

static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type,
                                        const struct bpf_insn *si,
                                        struct bpf_insn *insn_buf,
                                        struct bpf_prog *prog,
                                        u32 *target_size)
{
        struct bpf_insn *insn = insn_buf;

        switch (si->off) {
        case offsetof(struct bpf_sk_lookup, sk):
                *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sk_lookup_kern, selected_sk));
                break;

        case offsetof(struct bpf_sk_lookup, family):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     family, 2, target_size));
                break;

        case offsetof(struct bpf_sk_lookup, protocol):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     protocol, 2, target_size));
                break;

        case offsetof(struct bpf_sk_lookup, remote_ip4):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     v4.saddr, 4, target_size));
                break;

        case offsetof(struct bpf_sk_lookup, local_ip4):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     v4.daddr, 4, target_size));
                break;

        case bpf_ctx_range_till(struct bpf_sk_lookup,
                                remote_ip6[0], remote_ip6[3]): {
#if IS_ENABLED(CONFIG_IPV6)
                int off = si->off;

                off -= offsetof(struct bpf_sk_lookup, remote_ip6[0]);
                off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
                *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sk_lookup_kern, v6.saddr));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;
        }
        case bpf_ctx_range_till(struct bpf_sk_lookup,
                                local_ip6[0], local_ip6[3]): {
#if IS_ENABLED(CONFIG_IPV6)
                int off = si->off;

                off -= offsetof(struct bpf_sk_lookup, local_ip6[0]);
                off += bpf_target_off(struct in6_addr, s6_addr32[0], 4, target_size);
                *insn++ = BPF_LDX_MEM(BPF_SIZEOF(void *), si->dst_reg, si->src_reg,
                                      offsetof(struct bpf_sk_lookup_kern, v6.daddr));
                *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->dst_reg, off);
#else
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
#endif
                break;
        }
        case offsetof(struct bpf_sk_lookup, remote_port):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     sport, 2, target_size));
                break;

        case offsetofend(struct bpf_sk_lookup, remote_port):
                *target_size = 2;
                *insn++ = BPF_MOV32_IMM(si->dst_reg, 0);
                break;

        case offsetof(struct bpf_sk_lookup, local_port):
                *insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     dport, 2, target_size));
                break;

        case offsetof(struct bpf_sk_lookup, ingress_ifindex):
                *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg,
                                      bpf_target_off(struct bpf_sk_lookup_kern,
                                                     ingress_ifindex, 4, target_size));
                break;
        }

        return insn - insn_buf;
}

const struct bpf_prog_ops sk_lookup_prog_ops = {
        .test_run = bpf_prog_test_run_sk_lookup,
};

const struct bpf_verifier_ops sk_lookup_verifier_ops = {
        .get_func_proto                = sk_lookup_func_proto,
        .is_valid_access        = sk_lookup_is_valid_access,
        .convert_ctx_access        = sk_lookup_convert_ctx_access,
};

#endif /* CONFIG_INET */

DEFINE_BPF_DISPATCHER(xdp)

void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog)
{
        bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), prev_prog, prog);
}

BTF_ID_LIST_GLOBAL(btf_sock_ids, MAX_BTF_SOCK_TYPE)
#define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type)
BTF_SOCK_TYPE_xxx
#undef BTF_SOCK_TYPE

BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk)
{
        /* tcp6_sock type is not generated in dwarf and hence btf,
         * trigger an explicit type generation here.
         */
        BTF_TYPE_EMIT(struct tcp6_sock);
        if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP &&
            sk->sk_family == AF_INET6)
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto = {
        .func                        = bpf_skc_to_tcp6_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_TCP6],
};

BPF_CALL_1(bpf_skc_to_tcp_sock, struct sock *, sk)
{
        if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_TCP)
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_tcp_sock_proto = {
        .func                        = bpf_skc_to_tcp_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_TCP],
};

BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
{
        /* BTF types for tcp_timewait_sock and inet_timewait_sock are not
         * generated if CONFIG_INET=n. Trigger an explicit generation here.
         */
        BTF_TYPE_EMIT(struct inet_timewait_sock);
        BTF_TYPE_EMIT(struct tcp_timewait_sock);

#ifdef CONFIG_INET
        if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
                return (unsigned long)sk;
#endif

#if IS_BUILTIN(CONFIG_IPV6)
        if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_TIME_WAIT)
                return (unsigned long)sk;
#endif

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
        .func                        = bpf_skc_to_tcp_timewait_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW],
};

BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
{
#ifdef CONFIG_INET
        if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
                return (unsigned long)sk;
#endif

#if IS_BUILTIN(CONFIG_IPV6)
        if (sk && sk->sk_prot == &tcpv6_prot && sk->sk_state == TCP_NEW_SYN_RECV)
                return (unsigned long)sk;
#endif

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto = {
        .func                        = bpf_skc_to_tcp_request_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ],
};

BPF_CALL_1(bpf_skc_to_udp6_sock, struct sock *, sk)
{
        /* udp6_sock type is not generated in dwarf and hence btf,
         * trigger an explicit type generation here.
         */
        BTF_TYPE_EMIT(struct udp6_sock);
        if (sk && sk_fullsock(sk) && sk->sk_protocol == IPPROTO_UDP &&
            sk->sk_type == SOCK_DGRAM && sk->sk_family == AF_INET6)
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_udp6_sock_proto = {
        .func                        = bpf_skc_to_udp6_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_UDP6],
};

BPF_CALL_1(bpf_skc_to_unix_sock, struct sock *, sk)
{
        /* unix_sock type is not generated in dwarf and hence btf,
         * trigger an explicit type generation here.
         */
        BTF_TYPE_EMIT(struct unix_sock);
        if (sk && sk_fullsock(sk) && sk->sk_family == AF_UNIX)
                return (unsigned long)sk;

        return (unsigned long)NULL;
}

const struct bpf_func_proto bpf_skc_to_unix_sock_proto = {
        .func                        = bpf_skc_to_unix_sock,
        .gpl_only                = false,
        .ret_type                = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type                = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
        .ret_btf_id                = &btf_sock_ids[BTF_SOCK_TYPE_UNIX],
};

BPF_CALL_1(bpf_skc_to_mptcp_sock, struct sock *, sk)
{
        BTF_TYPE_EMIT(struct mptcp_sock);
        return (unsigned long)bpf_mptcp_sock_from_subflow(sk);
}

const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto = {
        .func                = bpf_skc_to_mptcp_sock,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_BTF_ID_OR_NULL,
        .arg1_type        = ARG_PTR_TO_SOCK_COMMON,
        .ret_btf_id        = &btf_sock_ids[BTF_SOCK_TYPE_MPTCP],
};

BPF_CALL_1(bpf_sock_from_file, struct file *, file)
{
        return (unsigned long)sock_from_file(file);
}

BTF_ID_LIST(bpf_sock_from_file_btf_ids)
BTF_ID(struct, socket)
BTF_ID(struct, file)

const struct bpf_func_proto bpf_sock_from_file_proto = {
        .func                = bpf_sock_from_file,
        .gpl_only        = false,
        .ret_type        = RET_PTR_TO_BTF_ID_OR_NULL,
        .ret_btf_id        = &bpf_sock_from_file_btf_ids[0],
        .arg1_type        = ARG_PTR_TO_BTF_ID,
        .arg1_btf_id        = &bpf_sock_from_file_btf_ids[1],
};

static const struct bpf_func_proto *
bpf_sk_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_func_proto *func;

        switch (func_id) {
        case BPF_FUNC_skc_to_tcp6_sock:
                func = &bpf_skc_to_tcp6_sock_proto;
                break;
        case BPF_FUNC_skc_to_tcp_sock:
                func = &bpf_skc_to_tcp_sock_proto;
                break;
        case BPF_FUNC_skc_to_tcp_timewait_sock:
                func = &bpf_skc_to_tcp_timewait_sock_proto;
                break;
        case BPF_FUNC_skc_to_tcp_request_sock:
                func = &bpf_skc_to_tcp_request_sock_proto;
                break;
        case BPF_FUNC_skc_to_udp6_sock:
                func = &bpf_skc_to_udp6_sock_proto;
                break;
        case BPF_FUNC_skc_to_unix_sock:
                func = &bpf_skc_to_unix_sock_proto;
                break;
        case BPF_FUNC_skc_to_mptcp_sock:
                func = &bpf_skc_to_mptcp_sock_proto;
                break;
        case BPF_FUNC_ktime_get_coarse_ns:
                return &bpf_ktime_get_coarse_ns_proto;
        default:
                return bpf_base_func_proto(func_id, prog);
        }

        if (!bpf_token_capable(prog->aux->token, CAP_PERFMON))
                return NULL;

        return func;
}

__bpf_kfunc_start_defs();
__bpf_kfunc int bpf_dynptr_from_skb(struct __sk_buff *s, u64 flags,
                                    struct bpf_dynptr *ptr__uninit)
{
        struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
        struct sk_buff *skb = (struct sk_buff *)s;

        if (flags) {
                bpf_dynptr_set_null(ptr);
                return -EINVAL;
        }

        bpf_dynptr_init(ptr, skb, BPF_DYNPTR_TYPE_SKB, 0, skb->len);

        return 0;
}

__bpf_kfunc int bpf_dynptr_from_xdp(struct xdp_md *x, u64 flags,
                                    struct bpf_dynptr *ptr__uninit)
{
        struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
        struct xdp_buff *xdp = (struct xdp_buff *)x;

        if (flags) {
                bpf_dynptr_set_null(ptr);
                return -EINVAL;
        }

        bpf_dynptr_init(ptr, xdp, BPF_DYNPTR_TYPE_XDP, 0, xdp_get_buff_len(xdp));

        return 0;
}

__bpf_kfunc int bpf_sock_addr_set_sun_path(struct bpf_sock_addr_kern *sa_kern,
                                           const u8 *sun_path, u32 sun_path__sz)
{
        struct sockaddr_un *un;

        if (sa_kern->sk->sk_family != AF_UNIX)
                return -EINVAL;

        /* We do not allow changing the address to unnamed or larger than the
         * maximum allowed address size for a unix sockaddr.
         */
        if (sun_path__sz == 0 || sun_path__sz > UNIX_PATH_MAX)
                return -EINVAL;

        un = (struct sockaddr_un *)sa_kern->uaddr;
        memcpy(un->sun_path, sun_path, sun_path__sz);
        sa_kern->uaddrlen = offsetof(struct sockaddr_un, sun_path) + sun_path__sz;

        return 0;
}

__bpf_kfunc int bpf_sk_assign_tcp_reqsk(struct __sk_buff *s, struct sock *sk,
                                        struct bpf_tcp_req_attrs *attrs, int attrs__sz)
{
#if IS_ENABLED(CONFIG_SYN_COOKIES)
        struct sk_buff *skb = (struct sk_buff *)s;
        const struct request_sock_ops *ops;
        struct inet_request_sock *ireq;
        struct tcp_request_sock *treq;
        struct request_sock *req;
        struct net *net;
        __u16 min_mss;
        u32 tsoff = 0;

        if (attrs__sz != sizeof(*attrs) ||
            attrs->reserved[0] || attrs->reserved[1] || attrs->reserved[2])
                return -EINVAL;

        if (!skb_at_tc_ingress(skb))
                return -EINVAL;

        net = dev_net(skb->dev);
        if (net != sock_net(sk))
                return -ENETUNREACH;

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                ops = &tcp_request_sock_ops;
                min_mss = 536;
                break;
#if IS_BUILTIN(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                ops = &tcp6_request_sock_ops;
                min_mss = IPV6_MIN_MTU - 60;
                break;
#endif
        default:
                return -EINVAL;
        }

        if (sk->sk_type != SOCK_STREAM || sk->sk_state != TCP_LISTEN ||
            sk_is_mptcp(sk))
                return -EINVAL;

        if (attrs->mss < min_mss)
                return -EINVAL;

        if (attrs->wscale_ok) {
                if (!READ_ONCE(net->ipv4.sysctl_tcp_window_scaling))
                        return -EINVAL;

                if (attrs->snd_wscale > TCP_MAX_WSCALE ||
                    attrs->rcv_wscale > TCP_MAX_WSCALE)
                        return -EINVAL;
        }

        if (attrs->sack_ok && !READ_ONCE(net->ipv4.sysctl_tcp_sack))
                return -EINVAL;

        if (attrs->tstamp_ok) {
                if (!READ_ONCE(net->ipv4.sysctl_tcp_timestamps))
                        return -EINVAL;

                tsoff = attrs->rcv_tsecr - tcp_ns_to_ts(attrs->usec_ts_ok, tcp_clock_ns());
        }

        req = inet_reqsk_alloc(ops, sk, false);
        if (!req)
                return -ENOMEM;

        ireq = inet_rsk(req);
        treq = tcp_rsk(req);

        req->rsk_listener = sk;
        req->syncookie = 1;
        req->mss = attrs->mss;
        req->ts_recent = attrs->rcv_tsval;

        ireq->snd_wscale = attrs->snd_wscale;
        ireq->rcv_wscale = attrs->rcv_wscale;
        ireq->tstamp_ok        = !!attrs->tstamp_ok;
        ireq->sack_ok = !!attrs->sack_ok;
        ireq->wscale_ok = !!attrs->wscale_ok;
        ireq->ecn_ok = !!attrs->ecn_ok;

        treq->req_usec_ts = !!attrs->usec_ts_ok;
        treq->ts_off = tsoff;

        skb_orphan(skb);
        skb->sk = req_to_sk(req);
        skb->destructor = sock_pfree;

        return 0;
#else
        return -EOPNOTSUPP;
#endif
}

__bpf_kfunc int bpf_sock_ops_enable_tx_tstamp(struct bpf_sock_ops_kern *skops,
                                              u64 flags)
{
        struct sk_buff *skb;

        if (skops->op != BPF_SOCK_OPS_TSTAMP_SENDMSG_CB)
                return -EOPNOTSUPP;

        if (flags)
                return -EINVAL;

        skb = skops->skb;
        skb_shinfo(skb)->tx_flags |= SKBTX_BPF;
        TCP_SKB_CB(skb)->txstamp_ack |= TSTAMP_ACK_BPF;
        skb_shinfo(skb)->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;

        return 0;
}

__bpf_kfunc_end_defs();

int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags,
                               struct bpf_dynptr *ptr__uninit)
{
        struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)ptr__uninit;
        int err;

        err = bpf_dynptr_from_skb(skb, flags, ptr__uninit);
        if (err)
                return err;

        bpf_dynptr_set_rdonly(ptr);

        return 0;
}

BTF_KFUNCS_START(bpf_kfunc_check_set_skb)
BTF_ID_FLAGS(func, bpf_dynptr_from_skb, KF_TRUSTED_ARGS)
BTF_KFUNCS_END(bpf_kfunc_check_set_skb)

BTF_KFUNCS_START(bpf_kfunc_check_set_xdp)
BTF_ID_FLAGS(func, bpf_dynptr_from_xdp)
BTF_KFUNCS_END(bpf_kfunc_check_set_xdp)

BTF_KFUNCS_START(bpf_kfunc_check_set_sock_addr)
BTF_ID_FLAGS(func, bpf_sock_addr_set_sun_path)
BTF_KFUNCS_END(bpf_kfunc_check_set_sock_addr)

BTF_KFUNCS_START(bpf_kfunc_check_set_tcp_reqsk)
BTF_ID_FLAGS(func, bpf_sk_assign_tcp_reqsk, KF_TRUSTED_ARGS)
BTF_KFUNCS_END(bpf_kfunc_check_set_tcp_reqsk)

BTF_KFUNCS_START(bpf_kfunc_check_set_sock_ops)
BTF_ID_FLAGS(func, bpf_sock_ops_enable_tx_tstamp, KF_TRUSTED_ARGS)
BTF_KFUNCS_END(bpf_kfunc_check_set_sock_ops)

static const struct btf_kfunc_id_set bpf_kfunc_set_skb = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_skb,
};

static const struct btf_kfunc_id_set bpf_kfunc_set_xdp = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_xdp,
};

static const struct btf_kfunc_id_set bpf_kfunc_set_sock_addr = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_sock_addr,
};

static const struct btf_kfunc_id_set bpf_kfunc_set_tcp_reqsk = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_tcp_reqsk,
};

static const struct btf_kfunc_id_set bpf_kfunc_set_sock_ops = {
        .owner = THIS_MODULE,
        .set = &bpf_kfunc_check_set_sock_ops,
};

static int __init bpf_kfunc_init(void)
{
        int ret;

        ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_ACT, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SK_SKB, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCKET_FILTER, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_OUT, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_IN, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_XMIT, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_LWT_SEG6LOCAL, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_NETFILTER, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_kfunc_set_skb);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &bpf_kfunc_set_xdp);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
                                               &bpf_kfunc_set_sock_addr);
        ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_kfunc_set_tcp_reqsk);
        return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SOCK_OPS, &bpf_kfunc_set_sock_ops);
}
late_initcall(bpf_kfunc_init);

__bpf_kfunc_start_defs();

/* bpf_sock_destroy: Destroy the given socket with ECONNABORTED error code.
 *
 * The function expects a non-NULL pointer to a socket, and invokes the
 * protocol specific socket destroy handlers.
 *
 * The helper can only be called from BPF contexts that have acquired the socket
 * locks.
 *
 * Parameters:
 * @sock: Pointer to socket to be destroyed
 *
 * Return:
 * On error, may return EPROTONOSUPPORT, EINVAL.
 * EPROTONOSUPPORT if protocol specific destroy handler is not supported.
 * 0 otherwise
 */
__bpf_kfunc int bpf_sock_destroy(struct sock_common *sock)
{
        struct sock *sk = (struct sock *)sock;

        /* The locking semantics that allow for synchronous execution of the
         * destroy handlers are only supported for TCP and UDP.
         * Supporting protocols will need to acquire sock lock in the BPF context
         * prior to invoking this kfunc.
         */
        if (!sk->sk_prot->diag_destroy || (sk->sk_protocol != IPPROTO_TCP &&
                                           sk->sk_protocol != IPPROTO_UDP))
                return -EOPNOTSUPP;

        return sk->sk_prot->diag_destroy(sk, ECONNABORTED);
}

__bpf_kfunc_end_defs();

BTF_KFUNCS_START(bpf_sk_iter_kfunc_ids)
BTF_ID_FLAGS(func, bpf_sock_destroy, KF_TRUSTED_ARGS)
BTF_KFUNCS_END(bpf_sk_iter_kfunc_ids)

static int tracing_iter_filter(const struct bpf_prog *prog, u32 kfunc_id)
{
        if (btf_id_set8_contains(&bpf_sk_iter_kfunc_ids, kfunc_id) &&
            prog->expected_attach_type != BPF_TRACE_ITER)
                return -EACCES;
        return 0;
}

static const struct btf_kfunc_id_set bpf_sk_iter_kfunc_set = {
        .owner = THIS_MODULE,
        .set   = &bpf_sk_iter_kfunc_ids,
        .filter = tracing_iter_filter,
};

static int init_subsystem(void)
{
        return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_sk_iter_kfunc_set);
}
late_initcall(init_subsystem);












































































































































  198 

















   86 






  293 





  294 




  240 




  240 






   16 




















   16 








   16 



   16 

   16 












   16 

































   16 








   15 


   15 


   16 
   16 




























   85 












   84 







   85 
   85 




















  261 





  266 



  239 










  259 






  259 






  262 
  261 








  265 




    1 




  122 
  122 



    4 









  251 





  250 

  250 













































   24 



   24 









   10 




























































































  193 
  187 































   33 




   48 





   48 



   48 




   48 

  193 


































   48 







    2 

   48 





   48 

   48 


   48 












   34 







   48 





  193 










  193 






























  193 
   48 






   48 














   48 





   48 




  193 











  193 




  193 







   20 
  193 
















































   24 





   24 




























































   16 


    9 






   19 


   19 


















   24 


   24 
    7 





















    1 





   24 






  269 
















    3 
















  268 














  268 


























   24 






















  273 




































































    5 























  269 



















    5 
    5 
    5 
    5 






    5 


















   24 
    7 











   24 

   24 




   24 
   24 


   24 

















   24 

    2 

   24 




   24 

   24 
   24 








   24 
   24 



   73 
  134 














   34 
   34 
   24 

















   26 









  163 



  160 

  163 





    4 
    7 

































  159 



  160 

  159 







































   91 
   12 


   25 











   94 




   83 

   80 












   19 






   24 



















  106 
  106 



   91 
   25 


   25 
    3 

   25 
   12 

















  106 









   91 
   24 


    7 
   95 







































  106 





  106 









  106 
    5 





















































   12 

   85 
   89 
    2 
    1 




   86 
    2 






    4 










  104 
  104 




   10 
   12 

   91 

   92 
   89 






   95 





   12 







    9 
    1 


























  106 










   22 


















   22 





































   91 











   12 












    7 












    3 






  107 
































   93 





   22 


  107 








    3 


    2 












   91 

   12 

    7 

    3 












  104 






   99 


   99 
    9 








  121 








  121 


    1 






    1 


    1 


    1 
    2 



  115 



    1 

    1 





















    4 





   12 





   93 






   93 





    8 
    1 



   11 

    6 





   99 





















   94 





    4 





















  122 
    1 


  121 





















































































    1 




    2 


    5 
    3 
    2 








    1 














    1 



    2 













    2 


    3 


























    8 




    8 
























    1 




    2 


    2 


   10 
    9 
    2 






    2 
    1 
    2 






    3 



    1 




    1 


    1 







    1 





    2 

    1 







   15 




   15 

















































































































































































































  285 
  322 





































   87 
   87 

   87 





































  265 





  238 


    2 





    6 





   11 





    4 




   10 
   10 



















  284 
  234 


  265 




























  243 

  243 




   33 
  254 



  259 



  258 
  259 




  243 



  242 









  258 

  235 
  246 



























  262 







  247 
   31 






























  247 








  234 



  229 

    1 





  244 





























































  262 









  262 


  248 







    9 












    9 





    9 














  250 
  236 









  261 














  263 



















































































































   90 






    6 


   89 







   91 
   91 















   91 





   88 

   12 

































































   23 






    4 


   19 
    7 

   12 





   23 
   23 
















   23 





   12 

   11 


















































































































































































  247 
  247 


  243 


    6 


  236 

    2 




   12 





    3 


    3 














  292 
  289 








    4 



  239 
  237 


    2 













    4 




























    9 



    6 





    9 




    9 









    9 








    9 


    9 

    6 









    6 












    3 



















    7 






    3 













    9 
    9 






    9 
    1 




    9 













    3 

    6 












    6 
    3 



    6 




    3 






    5 

    4 





    6 






    1 













   29 


    1 










    6 











   24 
















   28 
























































































































































































   40 








   95 




   87 






    9 

    5 





   90 









   42 



   42 
   41 








    2 


















   73 









































   80 













    1 










    2 


















   79 




    1 


   78 


    1 





    7 





























   73 






























    1 



    1 














    1 














    1 

















    1 



























































  681 




















  682 






    1 


    3 







  681 

















    1 









   19 




   14 


  285 





















    1 














    1 


    1 









    1 














    1 









    2 








    1 

    4 






    1 


    1 











    1 

    5 










    2 











    2 








    1 














    1 



    1 

















  418 




















































    2 


    2 












    1 


    6 


  240 




  247 


    2 




  186 

   43 

   29 

    2 







    7 



    6 

    1 



















































   66 






    1 




    2 















    5 



   61 


   61 


   61 














   61 




  116 
   17 
























    1 



    1 








    1 




    2 



    7 





















   87 










    1 




    1 


    1 




    1 




    1 



    1 








    3 




    1 




    1 


    1 

    1 












    6 


    6 


    6 









   23 
    7 







    1 

    1 




    2 

















    1 


    6 




    1 









    1 
    1 








   10 














    1 














    1 














    1 















  515 




    1 

  145 

   80 





    2 

   23 







    5 




  117 













  122 



  121 










    2 

    8 







    2 

   15 








    2 

    5 






    2 

   10 







    2 

   23 






    2 

   41 







    1 

   11 









    2 




   16 



    8 
    3 














    2 






    1 

    3 



   14 
    1 

























    4 



    7 



   62 






   44 


    3 


    1 














  103 















































































































  273 
    5 
















  269 













  364 




    1 




  273 


   65 


    1 










   28 


































































































































  269 




  269 












































   24 



































    5 
   12 






   24 


    5 










   30 







    7 
  135 










  116 
  116 




















   88 



  182 



  195 


  190 
   19 


    3 
   94 
   94 










  194 









  195 


  195 






































   87 
   13 


    6 
    1 
   86 










   92 









   92 


   92 






  155 




  155 

  155 


















   52 
  126 





  155 










   17 



   17 

   17 
    3 


    6 
   11 




    1 
























   11 






    8 






    8 



    8 
    7 





    8 















































































































































































































  288 







   24 
  269 
















  269 


   24 



    3 





  287 















































  292 




































  359 




  364 


































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Kernel-based Virtual Machine (KVM) Hypervisor
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 */

#include <kvm/iodev.h>

#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/miscdevice.h>
#include <linux/vmalloc.h>
#include <linux/reboot.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/file.h>
#include <linux/syscore_ops.h>
#include <linux/cpu.h>
#include <linux/sched/signal.h>
#include <linux/sched/mm.h>
#include <linux/sched/stat.h>
#include <linux/cpumask.h>
#include <linux/smp.h>
#include <linux/anon_inodes.h>
#include <linux/profile.h>
#include <linux/kvm_para.h>
#include <linux/pagemap.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/bitops.h>
#include <linux/spinlock.h>
#include <linux/compat.h>
#include <linux/srcu.h>
#include <linux/hugetlb.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/bsearch.h>
#include <linux/io.h>
#include <linux/lockdep.h>
#include <linux/kthread.h>
#include <linux/suspend.h>

#include <asm/processor.h>
#include <asm/ioctl.h>
#include <linux/uaccess.h>

#include "coalesced_mmio.h"
#include "async_pf.h"
#include "kvm_mm.h"
#include "vfio.h"

#include <trace/events/ipi.h>

#define CREATE_TRACE_POINTS
#include <trace/events/kvm.h>

#include <linux/kvm_dirty_ring.h>


/* Worst case buffer size needed for holding an integer. */
#define ITOA_MAX_LEN 12

MODULE_AUTHOR("Qumranet");
MODULE_DESCRIPTION("Kernel-based Virtual Machine (KVM) Hypervisor");
MODULE_LICENSE("GPL");

/* Architectures should define their poll value according to the halt latency */
unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
module_param(halt_poll_ns, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns);

/* Default doubles per-vcpu halt_poll_ns. */
unsigned int halt_poll_ns_grow = 2;
module_param(halt_poll_ns_grow, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns_grow);

/* The start value to grow halt_poll_ns from */
unsigned int halt_poll_ns_grow_start = 10000; /* 10us */
module_param(halt_poll_ns_grow_start, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start);

/* Default halves per-vcpu halt_poll_ns. */
unsigned int halt_poll_ns_shrink = 2;
module_param(halt_poll_ns_shrink, uint, 0644);
EXPORT_SYMBOL_GPL(halt_poll_ns_shrink);

/*
 * Allow direct access (from KVM or the CPU) without MMU notifier protection
 * to unpinned pages.
 */
static bool allow_unsafe_mappings;
module_param(allow_unsafe_mappings, bool, 0444);

/*
 * Ordering of locks:
 *
 *        kvm->lock --> kvm->slots_lock --> kvm->irq_lock
 */

DEFINE_MUTEX(kvm_lock);
LIST_HEAD(vm_list);

static struct kmem_cache *kvm_vcpu_cache;

static __read_mostly struct preempt_ops kvm_preempt_ops;
static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu);

static struct dentry *kvm_debugfs_dir;

static const struct file_operations stat_fops_per_vm;

static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
                           unsigned long arg);
#ifdef CONFIG_KVM_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
                                  unsigned long arg);
#define KVM_COMPAT(c)        .compat_ioctl        = (c)
#else
/*
 * For architectures that don't implement a compat infrastructure,
 * adopt a double line of defense:
 * - Prevent a compat task from opening /dev/kvm
 * - If the open has been done by a 64bit task, and the KVM fd
 *   passed to a compat task, let the ioctls fail.
 */
static long kvm_no_compat_ioctl(struct file *file, unsigned int ioctl,
                                unsigned long arg) { return -EINVAL; }

static int kvm_no_compat_open(struct inode *inode, struct file *file)
{
        return is_compat_task() ? -ENODEV : 0;
}
#define KVM_COMPAT(c)        .compat_ioctl        = kvm_no_compat_ioctl,        \
                        .open                = kvm_no_compat_open
#endif

static void kvm_io_bus_destroy(struct kvm_io_bus *bus);

#define KVM_EVENT_CREATE_VM 0
#define KVM_EVENT_DESTROY_VM 1
static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm);
static unsigned long long kvm_createvm_count;
static unsigned long long kvm_active_vms;

static DEFINE_PER_CPU(cpumask_var_t, cpu_kick_mask);

__weak void kvm_arch_guest_memory_reclaimed(struct kvm *kvm)
{
}

/*
 * Switches to specified vcpu, until a matching vcpu_put()
 */
void vcpu_load(struct kvm_vcpu *vcpu)
{
        int cpu = get_cpu();

        __this_cpu_write(kvm_running_vcpu, vcpu);
        preempt_notifier_register(&vcpu->preempt_notifier);
        kvm_arch_vcpu_load(vcpu, cpu);
        put_cpu();
}
EXPORT_SYMBOL_GPL(vcpu_load);

void vcpu_put(struct kvm_vcpu *vcpu)
{
        preempt_disable();
        kvm_arch_vcpu_put(vcpu);
        preempt_notifier_unregister(&vcpu->preempt_notifier);
        __this_cpu_write(kvm_running_vcpu, NULL);
        preempt_enable();
}
EXPORT_SYMBOL_GPL(vcpu_put);

/* TODO: merge with kvm_arch_vcpu_should_kick */
static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req)
{
        int mode = kvm_vcpu_exiting_guest_mode(vcpu);

        /*
         * We need to wait for the VCPU to reenable interrupts and get out of
         * READING_SHADOW_PAGE_TABLES mode.
         */
        if (req & KVM_REQUEST_WAIT)
                return mode != OUTSIDE_GUEST_MODE;

        /*
         * Need to kick a running VCPU, but otherwise there is nothing to do.
         */
        return mode == IN_GUEST_MODE;
}

static void ack_kick(void *_completed)
{
}

static inline bool kvm_kick_many_cpus(struct cpumask *cpus, bool wait)
{
        if (cpumask_empty(cpus))
                return false;

        smp_call_function_many(cpus, ack_kick, NULL, wait);
        return true;
}

static void kvm_make_vcpu_request(struct kvm_vcpu *vcpu, unsigned int req,
                                  struct cpumask *tmp, int current_cpu)
{
        int cpu;

        if (likely(!(req & KVM_REQUEST_NO_ACTION)))
                __kvm_make_request(req, vcpu);

        if (!(req & KVM_REQUEST_NO_WAKEUP) && kvm_vcpu_wake_up(vcpu))
                return;

        /*
         * Note, the vCPU could get migrated to a different pCPU at any point
         * after kvm_request_needs_ipi(), which could result in sending an IPI
         * to the previous pCPU.  But, that's OK because the purpose of the IPI
         * is to ensure the vCPU returns to OUTSIDE_GUEST_MODE, which is
         * satisfied if the vCPU migrates. Entering READING_SHADOW_PAGE_TABLES
         * after this point is also OK, as the requirement is only that KVM wait
         * for vCPUs that were reading SPTEs _before_ any changes were
         * finalized. See kvm_vcpu_kick() for more details on handling requests.
         */
        if (kvm_request_needs_ipi(vcpu, req)) {
                cpu = READ_ONCE(vcpu->cpu);
                if (cpu != -1 && cpu != current_cpu)
                        __cpumask_set_cpu(cpu, tmp);
        }
}

bool kvm_make_vcpus_request_mask(struct kvm *kvm, unsigned int req,
                                 unsigned long *vcpu_bitmap)
{
        struct kvm_vcpu *vcpu;
        struct cpumask *cpus;
        int i, me;
        bool called;

        me = get_cpu();

        cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
        cpumask_clear(cpus);

        for_each_set_bit(i, vcpu_bitmap, KVM_MAX_VCPUS) {
                vcpu = kvm_get_vcpu(kvm, i);
                if (!vcpu)
                        continue;
                kvm_make_vcpu_request(vcpu, req, cpus, me);
        }

        called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
        put_cpu();

        return called;
}

bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
{
        struct kvm_vcpu *vcpu;
        struct cpumask *cpus;
        unsigned long i;
        bool called;
        int me;

        me = get_cpu();

        cpus = this_cpu_cpumask_var_ptr(cpu_kick_mask);
        cpumask_clear(cpus);

        kvm_for_each_vcpu(i, vcpu, kvm)
                kvm_make_vcpu_request(vcpu, req, cpus, me);

        called = kvm_kick_many_cpus(cpus, !!(req & KVM_REQUEST_WAIT));
        put_cpu();

        return called;
}
EXPORT_SYMBOL_GPL(kvm_make_all_cpus_request);

void kvm_flush_remote_tlbs(struct kvm *kvm)
{
        ++kvm->stat.generic.remote_tlb_flush_requests;

        /*
         * We want to publish modifications to the page tables before reading
         * mode. Pairs with a memory barrier in arch-specific code.
         * - x86: smp_mb__after_srcu_read_unlock in vcpu_enter_guest
         * and smp_mb in walk_shadow_page_lockless_begin/end.
         * - powerpc: smp_mb in kvmppc_prepare_to_enter.
         *
         * There is already an smp_mb__after_atomic() before
         * kvm_make_all_cpus_request() reads vcpu->mode. We reuse that
         * barrier here.
         */
        if (!kvm_arch_flush_remote_tlbs(kvm)
            || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
                ++kvm->stat.generic.remote_tlb_flush;
}
EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);

void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
{
        if (!kvm_arch_flush_remote_tlbs_range(kvm, gfn, nr_pages))
                return;

        /*
         * Fall back to a flushing entire TLBs if the architecture range-based
         * TLB invalidation is unsupported or can't be performed for whatever
         * reason.
         */
        kvm_flush_remote_tlbs(kvm);
}

void kvm_flush_remote_tlbs_memslot(struct kvm *kvm,
                                   const struct kvm_memory_slot *memslot)
{
        /*
         * All current use cases for flushing the TLBs for a specific memslot
         * are related to dirty logging, and many do the TLB flush out of
         * mmu_lock. The interaction between the various operations on memslot
         * must be serialized by slots_locks to ensure the TLB flush from one
         * operation is observed by any other operation on the same memslot.
         */
        lockdep_assert_held(&kvm->slots_lock);
        kvm_flush_remote_tlbs_range(kvm, memslot->base_gfn, memslot->npages);
}

static void kvm_flush_shadow_all(struct kvm *kvm)
{
        kvm_arch_flush_shadow_all(kvm);
        kvm_arch_guest_memory_reclaimed(kvm);
}

#ifdef KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE
static inline void *mmu_memory_cache_alloc_obj(struct kvm_mmu_memory_cache *mc,
                                               gfp_t gfp_flags)
{
        void *page;

        gfp_flags |= mc->gfp_zero;

        if (mc->kmem_cache)
                return kmem_cache_alloc(mc->kmem_cache, gfp_flags);

        page = (void *)__get_free_page(gfp_flags);
        if (page && mc->init_value)
                memset64(page, mc->init_value, PAGE_SIZE / sizeof(u64));
        return page;
}

int __kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int capacity, int min)
{
        gfp_t gfp = mc->gfp_custom ? mc->gfp_custom : GFP_KERNEL_ACCOUNT;
        void *obj;

        if (mc->nobjs >= min)
                return 0;

        if (unlikely(!mc->objects)) {
                if (WARN_ON_ONCE(!capacity))
                        return -EIO;

                /*
                 * Custom init values can be used only for page allocations,
                 * and obviously conflict with __GFP_ZERO.
                 */
                if (WARN_ON_ONCE(mc->init_value && (mc->kmem_cache || mc->gfp_zero)))
                        return -EIO;

                mc->objects = kvmalloc_array(capacity, sizeof(void *), gfp);
                if (!mc->objects)
                        return -ENOMEM;

                mc->capacity = capacity;
        }

        /* It is illegal to request a different capacity across topups. */
        if (WARN_ON_ONCE(mc->capacity != capacity))
                return -EIO;

        while (mc->nobjs < mc->capacity) {
                obj = mmu_memory_cache_alloc_obj(mc, gfp);
                if (!obj)
                        return mc->nobjs >= min ? 0 : -ENOMEM;
                mc->objects[mc->nobjs++] = obj;
        }
        return 0;
}

int kvm_mmu_topup_memory_cache(struct kvm_mmu_memory_cache *mc, int min)
{
        return __kvm_mmu_topup_memory_cache(mc, KVM_ARCH_NR_OBJS_PER_MEMORY_CACHE, min);
}

int kvm_mmu_memory_cache_nr_free_objects(struct kvm_mmu_memory_cache *mc)
{
        return mc->nobjs;
}

void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
{
        while (mc->nobjs) {
                if (mc->kmem_cache)
                        kmem_cache_free(mc->kmem_cache, mc->objects[--mc->nobjs]);
                else
                        free_page((unsigned long)mc->objects[--mc->nobjs]);
        }

        kvfree(mc->objects);

        mc->objects = NULL;
        mc->capacity = 0;
}

void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
{
        void *p;

        if (WARN_ON(!mc->nobjs))
                p = mmu_memory_cache_alloc_obj(mc, GFP_ATOMIC | __GFP_ACCOUNT);
        else
                p = mc->objects[--mc->nobjs];
        BUG_ON(!p);
        return p;
}
#endif

static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
{
        mutex_init(&vcpu->mutex);
        vcpu->cpu = -1;
        vcpu->kvm = kvm;
        vcpu->vcpu_id = id;
        vcpu->pid = NULL;
        rwlock_init(&vcpu->pid_lock);
#ifndef __KVM_HAVE_ARCH_WQP
        rcuwait_init(&vcpu->wait);
#endif
        kvm_async_pf_vcpu_init(vcpu);

        kvm_vcpu_set_in_spin_loop(vcpu, false);
        kvm_vcpu_set_dy_eligible(vcpu, false);
        vcpu->preempted = false;
        vcpu->ready = false;
        preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
        vcpu->last_used_slot = NULL;

        /* Fill the stats id string for the vcpu */
        snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d",
                 task_pid_nr(current), id);
}

static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
{
        kvm_arch_vcpu_destroy(vcpu);
        kvm_dirty_ring_free(&vcpu->dirty_ring);

        /*
         * No need for rcu_read_lock as VCPU_RUN is the only place that changes
         * the vcpu->pid pointer, and at destruction time all file descriptors
         * are already gone.
         */
        put_pid(vcpu->pid);

        free_page((unsigned long)vcpu->run);
        kmem_cache_free(kvm_vcpu_cache, vcpu);
}

void kvm_destroy_vcpus(struct kvm *kvm)
{
        unsigned long i;
        struct kvm_vcpu *vcpu;

        kvm_for_each_vcpu(i, vcpu, kvm) {
                kvm_vcpu_destroy(vcpu);
                xa_erase(&kvm->vcpu_array, i);

                /*
                 * Assert that the vCPU isn't visible in any way, to ensure KVM
                 * doesn't trigger a use-after-free if destroying vCPUs results
                 * in VM-wide request, e.g. to flush remote TLBs when tearing
                 * down MMUs, or to mark the VM dead if a KVM_BUG_ON() fires.
                 */
                WARN_ON_ONCE(xa_load(&kvm->vcpu_array, i) || kvm_get_vcpu(kvm, i));
        }

        atomic_set(&kvm->online_vcpus, 0);
}
EXPORT_SYMBOL_GPL(kvm_destroy_vcpus);

#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
{
        return container_of(mn, struct kvm, mmu_notifier);
}

typedef bool (*gfn_handler_t)(struct kvm *kvm, struct kvm_gfn_range *range);

typedef void (*on_lock_fn_t)(struct kvm *kvm);

struct kvm_mmu_notifier_range {
        /*
         * 64-bit addresses, as KVM notifiers can operate on host virtual
         * addresses (unsigned long) and guest physical addresses (64-bit).
         */
        u64 start;
        u64 end;
        union kvm_mmu_notifier_arg arg;
        gfn_handler_t handler;
        on_lock_fn_t on_lock;
        bool flush_on_ret;
        bool may_block;
        bool lockless;
};

/*
 * The inner-most helper returns a tuple containing the return value from the
 * arch- and action-specific handler, plus a flag indicating whether or not at
 * least one memslot was found, i.e. if the handler found guest memory.
 *
 * Note, most notifiers are averse to booleans, so even though KVM tracks the
 * return from arch code as a bool, outer helpers will cast it to an int. :-(
 */
typedef struct kvm_mmu_notifier_return {
        bool ret;
        bool found_memslot;
} kvm_mn_ret_t;

/*
 * Use a dedicated stub instead of NULL to indicate that there is no callback
 * function/handler.  The compiler technically can't guarantee that a real
 * function will have a non-zero address, and so it will generate code to
 * check for !NULL, whereas comparing against a stub will be elided at compile
 * time (unless the compiler is getting long in the tooth, e.g. gcc 4.9).
 */
static void kvm_null_fn(void)
{

}
#define IS_KVM_NULL_FN(fn) ((fn) == (void *)kvm_null_fn)

/* Iterate over each memslot intersecting [start, last] (inclusive) range */
#define kvm_for_each_memslot_in_hva_range(node, slots, start, last)             \
        for (node = interval_tree_iter_first(&slots->hva_tree, start, last); \
             node;                                                             \
             node = interval_tree_iter_next(node, start, last))             \

static __always_inline kvm_mn_ret_t kvm_handle_hva_range(struct kvm *kvm,
                                                         const struct kvm_mmu_notifier_range *range)
{
        struct kvm_mmu_notifier_return r = {
                .ret = false,
                .found_memslot = false,
        };
        struct kvm_gfn_range gfn_range;
        struct kvm_memory_slot *slot;
        struct kvm_memslots *slots;
        int i, idx;

        if (WARN_ON_ONCE(range->end <= range->start))
                return r;

        /* A null handler is allowed if and only if on_lock() is provided. */
        if (WARN_ON_ONCE(IS_KVM_NULL_FN(range->on_lock) &&
                         IS_KVM_NULL_FN(range->handler)))
                return r;

        /* on_lock will never be called for lockless walks */
        if (WARN_ON_ONCE(range->lockless && !IS_KVM_NULL_FN(range->on_lock)))
                return r;

        idx = srcu_read_lock(&kvm->srcu);

        for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
                struct interval_tree_node *node;

                slots = __kvm_memslots(kvm, i);
                kvm_for_each_memslot_in_hva_range(node, slots,
                                                  range->start, range->end - 1) {
                        unsigned long hva_start, hva_end;

                        slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
                        hva_start = max_t(unsigned long, range->start, slot->userspace_addr);
                        hva_end = min_t(unsigned long, range->end,
                                        slot->userspace_addr + (slot->npages << PAGE_SHIFT));

                        /*
                         * To optimize for the likely case where the address
                         * range is covered by zero or one memslots, don't
                         * bother making these conditional (to avoid writes on
                         * the second or later invocation of the handler).
                         */
                        gfn_range.arg = range->arg;
                        gfn_range.may_block = range->may_block;
                        /*
                         * HVA-based notifications aren't relevant to private
                         * mappings as they don't have a userspace mapping.
                         */
                        gfn_range.attr_filter = KVM_FILTER_SHARED;

                        /*
                         * {gfn(page) | page intersects with [hva_start, hva_end)} =
                         * {gfn_start, gfn_start+1, ..., gfn_end-1}.
                         */
                        gfn_range.start = hva_to_gfn_memslot(hva_start, slot);
                        gfn_range.end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, slot);
                        gfn_range.slot = slot;
                        gfn_range.lockless = range->lockless;

                        if (!r.found_memslot) {
                                r.found_memslot = true;
                                if (!range->lockless) {
                                        KVM_MMU_LOCK(kvm);
                                        if (!IS_KVM_NULL_FN(range->on_lock))
                                                range->on_lock(kvm);

                                        if (IS_KVM_NULL_FN(range->handler))
                                                goto mmu_unlock;
                                }
                        }
                        r.ret |= range->handler(kvm, &gfn_range);
                }
        }

        if (range->flush_on_ret && r.ret)
                kvm_flush_remote_tlbs(kvm);

mmu_unlock:
        if (r.found_memslot && !range->lockless)
                KVM_MMU_UNLOCK(kvm);

        srcu_read_unlock(&kvm->srcu, idx);

        return r;
}

static __always_inline int kvm_age_hva_range(struct mmu_notifier *mn,
                                                unsigned long start,
                                                unsigned long end,
                                                gfn_handler_t handler,
                                                bool flush_on_ret)
{
        struct kvm *kvm = mmu_notifier_to_kvm(mn);
        const struct kvm_mmu_notifier_range range = {
                .start                = start,
                .end                = end,
                .handler        = handler,
                .on_lock        = (void *)kvm_null_fn,
                .flush_on_ret        = flush_on_ret,
                .may_block        = false,
                .lockless        = IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING),
        };

        return kvm_handle_hva_range(kvm, &range).ret;
}

static __always_inline int kvm_age_hva_range_no_flush(struct mmu_notifier *mn,
                                                      unsigned long start,
                                                      unsigned long end,
                                                      gfn_handler_t handler)
{
        return kvm_age_hva_range(mn, start, end, handler, false);
}

void kvm_mmu_invalidate_begin(struct kvm *kvm)
{
        lockdep_assert_held_write(&kvm->mmu_lock);
        /*
         * The count increase must become visible at unlock time as no
         * spte can be established without taking the mmu_lock and
         * count is also read inside the mmu_lock critical section.
         */
        kvm->mmu_invalidate_in_progress++;

        if (likely(kvm->mmu_invalidate_in_progress == 1)) {
                kvm->mmu_invalidate_range_start = INVALID_GPA;
                kvm->mmu_invalidate_range_end = INVALID_GPA;
        }
}

void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end)
{
        lockdep_assert_held_write(&kvm->mmu_lock);

        WARN_ON_ONCE(!kvm->mmu_invalidate_in_progress);

        if (likely(kvm->mmu_invalidate_range_start == INVALID_GPA)) {
                kvm->mmu_invalidate_range_start = start;
                kvm->mmu_invalidate_range_end = end;
        } else {
                /*
                 * Fully tracking multiple concurrent ranges has diminishing
                 * returns. Keep things simple and just find the minimal range
                 * which includes the current and new ranges. As there won't be
                 * enough information to subtract a range after its invalidate
                 * completes, any ranges invalidated concurrently will
                 * accumulate and persist until all outstanding invalidates
                 * complete.
                 */
                kvm->mmu_invalidate_range_start =
                        min(kvm->mmu_invalidate_range_start, start);
                kvm->mmu_invalidate_range_end =
                        max(kvm->mmu_invalidate_range_end, end);
        }
}

bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
{
        kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
        return kvm_unmap_gfn_range(kvm, range);
}

static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
                                        const struct mmu_notifier_range *range)
{
        struct kvm *kvm = mmu_notifier_to_kvm(mn);
        const struct kvm_mmu_notifier_range hva_range = {
                .start                = range->start,
                .end                = range->end,
                .handler        = kvm_mmu_unmap_gfn_range,
                .on_lock        = kvm_mmu_invalidate_begin,
                .flush_on_ret        = true,
                .may_block        = mmu_notifier_range_blockable(range),
        };

        trace_kvm_unmap_hva_range(range->start, range->end);

        /*
         * Prevent memslot modification between range_start() and range_end()
         * so that conditionally locking provides the same result in both
         * functions.  Without that guarantee, the mmu_invalidate_in_progress
         * adjustments will be imbalanced.
         *
         * Pairs with the decrement in range_end().
         */
        spin_lock(&kvm->mn_invalidate_lock);
        kvm->mn_active_invalidate_count++;
        spin_unlock(&kvm->mn_invalidate_lock);

        /*
         * Invalidate pfn caches _before_ invalidating the secondary MMUs, i.e.
         * before acquiring mmu_lock, to avoid holding mmu_lock while acquiring
         * each cache's lock.  There are relatively few caches in existence at
         * any given time, and the caches themselves can check for hva overlap,
         * i.e. don't need to rely on memslot overlap checks for performance.
         * Because this runs without holding mmu_lock, the pfn caches must use
         * mn_active_invalidate_count (see above) instead of
         * mmu_invalidate_in_progress.
         */
        gfn_to_pfn_cache_invalidate_start(kvm, range->start, range->end);

        /*
         * If one or more memslots were found and thus zapped, notify arch code
         * that guest memory has been reclaimed.  This needs to be done *after*
         * dropping mmu_lock, as x86's reclaim path is slooooow.
         */
        if (kvm_handle_hva_range(kvm, &hva_range).found_memslot)
                kvm_arch_guest_memory_reclaimed(kvm);

        return 0;
}

void kvm_mmu_invalidate_end(struct kvm *kvm)
{
        lockdep_assert_held_write(&kvm->mmu_lock);

        /*
         * This sequence increase will notify the kvm page fault that
         * the page that is going to be mapped in the spte could have
         * been freed.
         */
        kvm->mmu_invalidate_seq++;
        smp_wmb();
        /*
         * The above sequence increase must be visible before the
         * below count decrease, which is ensured by the smp_wmb above
         * in conjunction with the smp_rmb in mmu_invalidate_retry().
         */
        kvm->mmu_invalidate_in_progress--;
        KVM_BUG_ON(kvm->mmu_invalidate_in_progress < 0, kvm);

        /*
         * Assert that at least one range was added between start() and end().
         * Not adding a range isn't fatal, but it is a KVM bug.
         */
        WARN_ON_ONCE(kvm->mmu_invalidate_range_start == INVALID_GPA);
}

static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
                                        const struct mmu_notifier_range *range)
{
        struct kvm *kvm = mmu_notifier_to_kvm(mn);
        const struct kvm_mmu_notifier_range hva_range = {
                .start                = range->start,
                .end                = range->end,
                .handler        = (void *)kvm_null_fn,
                .on_lock        = kvm_mmu_invalidate_end,
                .flush_on_ret        = false,
                .may_block        = mmu_notifier_range_blockable(range),
        };
        bool wake;

        kvm_handle_hva_range(kvm, &hva_range);

        /* Pairs with the increment in range_start(). */
        spin_lock(&kvm->mn_invalidate_lock);
        if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count))
                --kvm->mn_active_invalidate_count;
        wake = !kvm->mn_active_invalidate_count;
        spin_unlock(&kvm->mn_invalidate_lock);

        /*
         * There can only be one waiter, since the wait happens under
         * slots_lock.
         */
        if (wake)
                rcuwait_wake_up(&kvm->mn_memslots_update_rcuwait);
}

static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
                                              struct mm_struct *mm,
                                              unsigned long start,
                                              unsigned long end)
{
        trace_kvm_age_hva(start, end);

        return kvm_age_hva_range(mn, start, end, kvm_age_gfn,
                                 !IS_ENABLED(CONFIG_KVM_ELIDE_TLB_FLUSH_IF_YOUNG));
}

static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
                                        struct mm_struct *mm,
                                        unsigned long start,
                                        unsigned long end)
{
        trace_kvm_age_hva(start, end);

        /*
         * Even though we do not flush TLB, this will still adversely
         * affect performance on pre-Haswell Intel EPT, where there is
         * no EPT Access Bit to clear so that we have to tear down EPT
         * tables instead. If we find this unacceptable, we can always
         * add a parameter to kvm_age_hva so that it effectively doesn't
         * do anything on clear_young.
         *
         * Also note that currently we never issue secondary TLB flushes
         * from clear_young, leaving this job up to the regular system
         * cadence. If we find this inaccurate, we might come up with a
         * more sophisticated heuristic later.
         */
        return kvm_age_hva_range_no_flush(mn, start, end, kvm_age_gfn);
}

static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
                                       struct mm_struct *mm,
                                       unsigned long address)
{
        trace_kvm_test_age_hva(address);

        return kvm_age_hva_range_no_flush(mn, address, address + 1,
                                          kvm_test_age_gfn);
}

static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
                                     struct mm_struct *mm)
{
        struct kvm *kvm = mmu_notifier_to_kvm(mn);
        int idx;

        idx = srcu_read_lock(&kvm->srcu);
        kvm_flush_shadow_all(kvm);
        srcu_read_unlock(&kvm->srcu, idx);
}

static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
        .invalidate_range_start        = kvm_mmu_notifier_invalidate_range_start,
        .invalidate_range_end        = kvm_mmu_notifier_invalidate_range_end,
        .clear_flush_young        = kvm_mmu_notifier_clear_flush_young,
        .clear_young                = kvm_mmu_notifier_clear_young,
        .test_young                = kvm_mmu_notifier_test_young,
        .release                = kvm_mmu_notifier_release,
};

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
        kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
        return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
}

#else  /* !CONFIG_KVM_GENERIC_MMU_NOTIFIER */

static int kvm_init_mmu_notifier(struct kvm *kvm)
{
        return 0;
}

#endif /* CONFIG_KVM_GENERIC_MMU_NOTIFIER */

#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
static int kvm_pm_notifier_call(struct notifier_block *bl,
                                unsigned long state,
                                void *unused)
{
        struct kvm *kvm = container_of(bl, struct kvm, pm_notifier);

        return kvm_arch_pm_notifier(kvm, state);
}

static void kvm_init_pm_notifier(struct kvm *kvm)
{
        kvm->pm_notifier.notifier_call = kvm_pm_notifier_call;
        /* Suspend KVM before we suspend ftrace, RCU, etc. */
        kvm->pm_notifier.priority = INT_MAX;
        register_pm_notifier(&kvm->pm_notifier);
}

static void kvm_destroy_pm_notifier(struct kvm *kvm)
{
        unregister_pm_notifier(&kvm->pm_notifier);
}
#else /* !CONFIG_HAVE_KVM_PM_NOTIFIER */
static void kvm_init_pm_notifier(struct kvm *kvm)
{
}

static void kvm_destroy_pm_notifier(struct kvm *kvm)
{
}
#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */

static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
{
        if (!memslot->dirty_bitmap)
                return;

        vfree(memslot->dirty_bitmap);
        memslot->dirty_bitmap = NULL;
}

/* This does not remove the slot from struct kvm_memslots data structures */
static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
{
        if (slot->flags & KVM_MEM_GUEST_MEMFD)
                kvm_gmem_unbind(slot);

        kvm_destroy_dirty_bitmap(slot);

        kvm_arch_free_memslot(kvm, slot);

        kfree(slot);
}

static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
{
        struct hlist_node *idnode;
        struct kvm_memory_slot *memslot;
        int bkt;

        /*
         * The same memslot objects live in both active and inactive sets,
         * arbitrarily free using index '1' so the second invocation of this
         * function isn't operating over a structure with dangling pointers
         * (even though this function isn't actually touching them).
         */
        if (!slots->node_idx)
                return;

        hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1])
                kvm_free_memslot(kvm, memslot);
}

static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc)
{
        switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
        case KVM_STATS_TYPE_INSTANT:
                return 0444;
        case KVM_STATS_TYPE_CUMULATIVE:
        case KVM_STATS_TYPE_PEAK:
        default:
                return 0644;
        }
}


static void kvm_destroy_vm_debugfs(struct kvm *kvm)
{
        int i;
        int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
                                      kvm_vcpu_stats_header.num_desc;

        if (IS_ERR(kvm->debugfs_dentry))
                return;

        debugfs_remove_recursive(kvm->debugfs_dentry);

        if (kvm->debugfs_stat_data) {
                for (i = 0; i < kvm_debugfs_num_entries; i++)
                        kfree(kvm->debugfs_stat_data[i]);
                kfree(kvm->debugfs_stat_data);
        }
}

static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname)
{
        static DEFINE_MUTEX(kvm_debugfs_lock);
        struct dentry *dent;
        char dir_name[ITOA_MAX_LEN * 2];
        struct kvm_stat_data *stat_data;
        const struct _kvm_stats_desc *pdesc;
        int i, ret = -ENOMEM;
        int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc +
                                      kvm_vcpu_stats_header.num_desc;

        if (!debugfs_initialized())
                return 0;

        snprintf(dir_name, sizeof(dir_name), "%d-%s", task_pid_nr(current), fdname);
        mutex_lock(&kvm_debugfs_lock);
        dent = debugfs_lookup(dir_name, kvm_debugfs_dir);
        if (dent) {
                pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name);
                dput(dent);
                mutex_unlock(&kvm_debugfs_lock);
                return 0;
        }
        dent = debugfs_create_dir(dir_name, kvm_debugfs_dir);
        mutex_unlock(&kvm_debugfs_lock);
        if (IS_ERR(dent))
                return 0;

        kvm->debugfs_dentry = dent;
        kvm->debugfs_stat_data = kcalloc(kvm_debugfs_num_entries,
                                         sizeof(*kvm->debugfs_stat_data),
                                         GFP_KERNEL_ACCOUNT);
        if (!kvm->debugfs_stat_data)
                goto out_err;

        for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
                pdesc = &kvm_vm_stats_desc[i];
                stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
                if (!stat_data)
                        goto out_err;

                stat_data->kvm = kvm;
                stat_data->desc = pdesc;
                stat_data->kind = KVM_STAT_VM;
                kvm->debugfs_stat_data[i] = stat_data;
                debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                                    kvm->debugfs_dentry, stat_data,
                                    &stat_fops_per_vm);
        }

        for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
                pdesc = &kvm_vcpu_stats_desc[i];
                stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT);
                if (!stat_data)
                        goto out_err;

                stat_data->kvm = kvm;
                stat_data->desc = pdesc;
                stat_data->kind = KVM_STAT_VCPU;
                kvm->debugfs_stat_data[i + kvm_vm_stats_header.num_desc] = stat_data;
                debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                                    kvm->debugfs_dentry, stat_data,
                                    &stat_fops_per_vm);
        }

        kvm_arch_create_vm_debugfs(kvm);
        return 0;
out_err:
        kvm_destroy_vm_debugfs(kvm);
        return ret;
}

/*
 * Called just after removing the VM from the vm_list, but before doing any
 * other destruction.
 */
void __weak kvm_arch_pre_destroy_vm(struct kvm *kvm)
{
}

/*
 * Called after per-vm debugfs created.  When called kvm->debugfs_dentry should
 * be setup already, so we can create arch-specific debugfs entries under it.
 * Cleanup should be automatic done in kvm_destroy_vm_debugfs() recursively, so
 * a per-arch destroy interface is not needed.
 */
void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
{
}

static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
{
        struct kvm *kvm = kvm_arch_alloc_vm();
        struct kvm_memslots *slots;
        int r, i, j;

        if (!kvm)
                return ERR_PTR(-ENOMEM);

        KVM_MMU_LOCK_INIT(kvm);
        mmgrab(current->mm);
        kvm->mm = current->mm;
        kvm_eventfd_init(kvm);
        mutex_init(&kvm->lock);
        mutex_init(&kvm->irq_lock);
        mutex_init(&kvm->slots_lock);
        mutex_init(&kvm->slots_arch_lock);
        spin_lock_init(&kvm->mn_invalidate_lock);
        rcuwait_init(&kvm->mn_memslots_update_rcuwait);
        xa_init(&kvm->vcpu_array);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
        xa_init(&kvm->mem_attr_array);
#endif

        INIT_LIST_HEAD(&kvm->gpc_list);
        spin_lock_init(&kvm->gpc_lock);

        INIT_LIST_HEAD(&kvm->devices);
        kvm->max_vcpus = KVM_MAX_VCPUS;

        BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);

        /*
         * Force subsequent debugfs file creations to fail if the VM directory
         * is not created (by kvm_create_vm_debugfs()).
         */
        kvm->debugfs_dentry = ERR_PTR(-ENOENT);

        snprintf(kvm->stats_id, sizeof(kvm->stats_id), "kvm-%d",
                 task_pid_nr(current));

        r = -ENOMEM;
        if (init_srcu_struct(&kvm->srcu))
                goto out_err_no_srcu;
        if (init_srcu_struct(&kvm->irq_srcu))
                goto out_err_no_irq_srcu;

        r = kvm_init_irq_routing(kvm);
        if (r)
                goto out_err_no_irq_routing;

        refcount_set(&kvm->users_count, 1);

        for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
                for (j = 0; j < 2; j++) {
                        slots = &kvm->__memslots[i][j];

                        atomic_long_set(&slots->last_used_slot, (unsigned long)NULL);
                        slots->hva_tree = RB_ROOT_CACHED;
                        slots->gfn_tree = RB_ROOT;
                        hash_init(slots->id_hash);
                        slots->node_idx = j;

                        /* Generations must be different for each address space. */
                        slots->generation = i;
                }

                rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
        }

        r = -ENOMEM;
        for (i = 0; i < KVM_NR_BUSES; i++) {
                rcu_assign_pointer(kvm->buses[i],
                        kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
                if (!kvm->buses[i])
                        goto out_err_no_arch_destroy_vm;
        }

        r = kvm_arch_init_vm(kvm, type);
        if (r)
                goto out_err_no_arch_destroy_vm;

        r = kvm_enable_virtualization();
        if (r)
                goto out_err_no_disable;

#ifdef CONFIG_HAVE_KVM_IRQCHIP
        INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
#endif

        r = kvm_init_mmu_notifier(kvm);
        if (r)
                goto out_err_no_mmu_notifier;

        r = kvm_coalesced_mmio_init(kvm);
        if (r < 0)
                goto out_no_coalesced_mmio;

        r = kvm_create_vm_debugfs(kvm, fdname);
        if (r)
                goto out_err_no_debugfs;

        mutex_lock(&kvm_lock);
        list_add(&kvm->vm_list, &vm_list);
        mutex_unlock(&kvm_lock);

        preempt_notifier_inc();
        kvm_init_pm_notifier(kvm);

        return kvm;

out_err_no_debugfs:
        kvm_coalesced_mmio_free(kvm);
out_no_coalesced_mmio:
#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
        if (kvm->mmu_notifier.ops)
                mmu_notifier_unregister(&kvm->mmu_notifier, current->mm);
#endif
out_err_no_mmu_notifier:
        kvm_disable_virtualization();
out_err_no_disable:
        kvm_arch_destroy_vm(kvm);
out_err_no_arch_destroy_vm:
        WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
        for (i = 0; i < KVM_NR_BUSES; i++)
                kfree(kvm_get_bus(kvm, i));
        kvm_free_irq_routing(kvm);
out_err_no_irq_routing:
        cleanup_srcu_struct(&kvm->irq_srcu);
out_err_no_irq_srcu:
        cleanup_srcu_struct(&kvm->srcu);
out_err_no_srcu:
        kvm_arch_free_vm(kvm);
        mmdrop(current->mm);
        return ERR_PTR(r);
}

static void kvm_destroy_devices(struct kvm *kvm)
{
        struct kvm_device *dev, *tmp;

        /*
         * We do not need to take the kvm->lock here, because nobody else
         * has a reference to the struct kvm at this point and therefore
         * cannot access the devices list anyhow.
         *
         * The device list is generally managed as an rculist, but list_del()
         * is used intentionally here. If a bug in KVM introduced a reader that
         * was not backed by a reference on the kvm struct, the hope is that
         * it'd consume the poisoned forward pointer instead of suffering a
         * use-after-free, even though this cannot be guaranteed.
         */
        list_for_each_entry_safe(dev, tmp, &kvm->devices, vm_node) {
                list_del(&dev->vm_node);
                dev->ops->destroy(dev);
        }
}

static void kvm_destroy_vm(struct kvm *kvm)
{
        int i;
        struct mm_struct *mm = kvm->mm;

        kvm_destroy_pm_notifier(kvm);
        kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm);
        kvm_destroy_vm_debugfs(kvm);
        mutex_lock(&kvm_lock);
        list_del(&kvm->vm_list);
        mutex_unlock(&kvm_lock);
        kvm_arch_pre_destroy_vm(kvm);

        kvm_free_irq_routing(kvm);
        for (i = 0; i < KVM_NR_BUSES; i++) {
                struct kvm_io_bus *bus = kvm_get_bus(kvm, i);

                if (bus)
                        kvm_io_bus_destroy(bus);
                kvm->buses[i] = NULL;
        }
        kvm_coalesced_mmio_free(kvm);
#ifdef CONFIG_KVM_GENERIC_MMU_NOTIFIER
        mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
        /*
         * At this point, pending calls to invalidate_range_start()
         * have completed but no more MMU notifiers will run, so
         * mn_active_invalidate_count may remain unbalanced.
         * No threads can be waiting in kvm_swap_active_memslots() as the
         * last reference on KVM has been dropped, but freeing
         * memslots would deadlock without this manual intervention.
         *
         * If the count isn't unbalanced, i.e. KVM did NOT unregister its MMU
         * notifier between a start() and end(), then there shouldn't be any
         * in-progress invalidations.
         */
        WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
        if (kvm->mn_active_invalidate_count)
                kvm->mn_active_invalidate_count = 0;
        else
                WARN_ON(kvm->mmu_invalidate_in_progress);
#else
        kvm_flush_shadow_all(kvm);
#endif
        kvm_arch_destroy_vm(kvm);
        kvm_destroy_devices(kvm);
        for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
                kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
                kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
        }
        cleanup_srcu_struct(&kvm->irq_srcu);
        cleanup_srcu_struct(&kvm->srcu);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
        xa_destroy(&kvm->mem_attr_array);
#endif
        kvm_arch_free_vm(kvm);
        preempt_notifier_dec();
        kvm_disable_virtualization();
        mmdrop(mm);
}

void kvm_get_kvm(struct kvm *kvm)
{
        refcount_inc(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm);

/*
 * Make sure the vm is not during destruction, which is a safe version of
 * kvm_get_kvm().  Return true if kvm referenced successfully, false otherwise.
 */
bool kvm_get_kvm_safe(struct kvm *kvm)
{
        return refcount_inc_not_zero(&kvm->users_count);
}
EXPORT_SYMBOL_GPL(kvm_get_kvm_safe);

void kvm_put_kvm(struct kvm *kvm)
{
        if (refcount_dec_and_test(&kvm->users_count))
                kvm_destroy_vm(kvm);
}
EXPORT_SYMBOL_GPL(kvm_put_kvm);

/*
 * Used to put a reference that was taken on behalf of an object associated
 * with a user-visible file descriptor, e.g. a vcpu or device, if installation
 * of the new file descriptor fails and the reference cannot be transferred to
 * its final owner.  In such cases, the caller is still actively using @kvm and
 * will fail miserably if the refcount unexpectedly hits zero.
 */
void kvm_put_kvm_no_destroy(struct kvm *kvm)
{
        WARN_ON(refcount_dec_and_test(&kvm->users_count));
}
EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy);

static int kvm_vm_release(struct inode *inode, struct file *filp)
{
        struct kvm *kvm = filp->private_data;

        kvm_irqfd_release(kvm);

        kvm_put_kvm(kvm);
        return 0;
}

int kvm_trylock_all_vcpus(struct kvm *kvm)
{
        struct kvm_vcpu *vcpu;
        unsigned long i, j;

        lockdep_assert_held(&kvm->lock);

        kvm_for_each_vcpu(i, vcpu, kvm)
                if (!mutex_trylock_nest_lock(&vcpu->mutex, &kvm->lock))
                        goto out_unlock;
        return 0;

out_unlock:
        kvm_for_each_vcpu(j, vcpu, kvm) {
                if (i == j)
                        break;
                mutex_unlock(&vcpu->mutex);
        }
        return -EINTR;
}
EXPORT_SYMBOL_GPL(kvm_trylock_all_vcpus);

int kvm_lock_all_vcpus(struct kvm *kvm)
{
        struct kvm_vcpu *vcpu;
        unsigned long i, j;
        int r;

        lockdep_assert_held(&kvm->lock);

        kvm_for_each_vcpu(i, vcpu, kvm) {
                r = mutex_lock_killable_nest_lock(&vcpu->mutex, &kvm->lock);
                if (r)
                        goto out_unlock;
        }
        return 0;

out_unlock:
        kvm_for_each_vcpu(j, vcpu, kvm) {
                if (i == j)
                        break;
                mutex_unlock(&vcpu->mutex);
        }
        return r;
}
EXPORT_SYMBOL_GPL(kvm_lock_all_vcpus);

void kvm_unlock_all_vcpus(struct kvm *kvm)
{
        struct kvm_vcpu *vcpu;
        unsigned long i;

        lockdep_assert_held(&kvm->lock);

        kvm_for_each_vcpu(i, vcpu, kvm)
                mutex_unlock(&vcpu->mutex);
}
EXPORT_SYMBOL_GPL(kvm_unlock_all_vcpus);

/*
 * Allocation size is twice as large as the actual dirty bitmap size.
 * See kvm_vm_ioctl_get_dirty_log() why this is needed.
 */
static int kvm_alloc_dirty_bitmap(struct kvm_memory_slot *memslot)
{
        unsigned long dirty_bytes = kvm_dirty_bitmap_bytes(memslot);

        memslot->dirty_bitmap = __vcalloc(2, dirty_bytes, GFP_KERNEL_ACCOUNT);
        if (!memslot->dirty_bitmap)
                return -ENOMEM;

        return 0;
}

static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)
{
        struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
        int node_idx_inactive = active->node_idx ^ 1;

        return &kvm->__memslots[as_id][node_idx_inactive];
}

/*
 * Helper to get the address space ID when one of memslot pointers may be NULL.
 * This also serves as a sanity that at least one of the pointers is non-NULL,
 * and that their address space IDs don't diverge.
 */
static int kvm_memslots_get_as_id(struct kvm_memory_slot *a,
                                  struct kvm_memory_slot *b)
{
        if (WARN_ON_ONCE(!a && !b))
                return 0;

        if (!a)
                return b->as_id;
        if (!b)
                return a->as_id;

        WARN_ON_ONCE(a->as_id != b->as_id);
        return a->as_id;
}

static void kvm_insert_gfn_node(struct kvm_memslots *slots,
                                struct kvm_memory_slot *slot)
{
        struct rb_root *gfn_tree = &slots->gfn_tree;
        struct rb_node **node, *parent;
        int idx = slots->node_idx;

        parent = NULL;
        for (node = &gfn_tree->rb_node; *node; ) {
                struct kvm_memory_slot *tmp;

                tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
                parent = *node;
                if (slot->base_gfn < tmp->base_gfn)
                        node = &(*node)->rb_left;
                else if (slot->base_gfn > tmp->base_gfn)
                        node = &(*node)->rb_right;
                else
                        BUG();
        }

        rb_link_node(&slot->gfn_node[idx], parent, node);
        rb_insert_color(&slot->gfn_node[idx], gfn_tree);
}

static void kvm_erase_gfn_node(struct kvm_memslots *slots,
                               struct kvm_memory_slot *slot)
{
        rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree);
}

static void kvm_replace_gfn_node(struct kvm_memslots *slots,
                                 struct kvm_memory_slot *old,
                                 struct kvm_memory_slot *new)
{
        int idx = slots->node_idx;

        WARN_ON_ONCE(old->base_gfn != new->base_gfn);

        rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx],
                        &slots->gfn_tree);
}

/*
 * Replace @old with @new in the inactive memslots.
 *
 * With NULL @old this simply adds @new.
 * With NULL @new this simply removes @old.
 *
 * If @new is non-NULL its hva_node[slots_idx] range has to be set
 * appropriately.
 */
static void kvm_replace_memslot(struct kvm *kvm,
                                struct kvm_memory_slot *old,
                                struct kvm_memory_slot *new)
{
        int as_id = kvm_memslots_get_as_id(old, new);
        struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
        int idx = slots->node_idx;

        if (old) {
                hash_del(&old->id_node[idx]);
                interval_tree_remove(&old->hva_node[idx], &slots->hva_tree);

                if ((long)old == atomic_long_read(&slots->last_used_slot))
                        atomic_long_set(&slots->last_used_slot, (long)new);

                if (!new) {
                        kvm_erase_gfn_node(slots, old);
                        return;
                }
        }

        /*
         * Initialize @new's hva range.  Do this even when replacing an @old
         * slot, kvm_copy_memslot() deliberately does not touch node data.
         */
        new->hva_node[idx].start = new->userspace_addr;
        new->hva_node[idx].last = new->userspace_addr +
                                  (new->npages << PAGE_SHIFT) - 1;

        /*
         * (Re)Add the new memslot.  There is no O(1) interval_tree_replace(),
         * hva_node needs to be swapped with remove+insert even though hva can't
         * change when replacing an existing slot.
         */
        hash_add(slots->id_hash, &new->id_node[idx], new->id);
        interval_tree_insert(&new->hva_node[idx], &slots->hva_tree);

        /*
         * If the memslot gfn is unchanged, rb_replace_node() can be used to
         * switch the node in the gfn tree instead of removing the old and
         * inserting the new as two separate operations. Replacement is a
         * single O(1) operation versus two O(log(n)) operations for
         * remove+insert.
         */
        if (old && old->base_gfn == new->base_gfn) {
                kvm_replace_gfn_node(slots, old, new);
        } else {
                if (old)
                        kvm_erase_gfn_node(slots, old);
                kvm_insert_gfn_node(slots, new);
        }
}

/*
 * Flags that do not access any of the extra space of struct
 * kvm_userspace_memory_region2.  KVM_SET_USER_MEMORY_REGION_V1_FLAGS
 * only allows these.
 */
#define KVM_SET_USER_MEMORY_REGION_V1_FLAGS \
        (KVM_MEM_LOG_DIRTY_PAGES | KVM_MEM_READONLY)

static int check_memory_region_flags(struct kvm *kvm,
                                     const struct kvm_userspace_memory_region2 *mem)
{
        u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;

        if (kvm_arch_has_private_mem(kvm))
                valid_flags |= KVM_MEM_GUEST_MEMFD;

        /* Dirty logging private memory is not currently supported. */
        if (mem->flags & KVM_MEM_GUEST_MEMFD)
                valid_flags &= ~KVM_MEM_LOG_DIRTY_PAGES;

        /*
         * GUEST_MEMFD is incompatible with read-only memslots, as writes to
         * read-only memslots have emulated MMIO, not page fault, semantics,
         * and KVM doesn't allow emulated MMIO for private memory.
         */
        if (kvm_arch_has_readonly_mem(kvm) &&
            !(mem->flags & KVM_MEM_GUEST_MEMFD))
                valid_flags |= KVM_MEM_READONLY;

        if (mem->flags & ~valid_flags)
                return -EINVAL;

        return 0;
}

static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
{
        struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);

        /* Grab the generation from the activate memslots. */
        u64 gen = __kvm_memslots(kvm, as_id)->generation;

        WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
        slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;

        /*
         * Do not store the new memslots while there are invalidations in
         * progress, otherwise the locking in invalidate_range_start and
         * invalidate_range_end will be unbalanced.
         */
        spin_lock(&kvm->mn_invalidate_lock);
        prepare_to_rcuwait(&kvm->mn_memslots_update_rcuwait);
        while (kvm->mn_active_invalidate_count) {
                set_current_state(TASK_UNINTERRUPTIBLE);
                spin_unlock(&kvm->mn_invalidate_lock);
                schedule();
                spin_lock(&kvm->mn_invalidate_lock);
        }
        finish_rcuwait(&kvm->mn_memslots_update_rcuwait);
        rcu_assign_pointer(kvm->memslots[as_id], slots);
        spin_unlock(&kvm->mn_invalidate_lock);

        /*
         * Acquired in kvm_set_memslot. Must be released before synchronize
         * SRCU below in order to avoid deadlock with another thread
         * acquiring the slots_arch_lock in an srcu critical section.
         */
        mutex_unlock(&kvm->slots_arch_lock);

        synchronize_srcu_expedited(&kvm->srcu);

        /*
         * Increment the new memslot generation a second time, dropping the
         * update in-progress flag and incrementing the generation based on
         * the number of address spaces.  This provides a unique and easily
         * identifiable generation number while the memslots are in flux.
         */
        gen = slots->generation & ~KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;

        /*
         * Generations must be unique even across address spaces.  We do not need
         * a global counter for that, instead the generation space is evenly split
         * across address spaces.  For example, with two address spaces, address
         * space 0 will use generations 0, 2, 4, ... while address space 1 will
         * use generations 1, 3, 5, ...
         */
        gen += kvm_arch_nr_memslot_as_ids(kvm);

        kvm_arch_memslots_updated(kvm, gen);

        slots->generation = gen;
}

static int kvm_prepare_memory_region(struct kvm *kvm,
                                     const struct kvm_memory_slot *old,
                                     struct kvm_memory_slot *new,
                                     enum kvm_mr_change change)
{
        int r;

        /*
         * If dirty logging is disabled, nullify the bitmap; the old bitmap
         * will be freed on "commit".  If logging is enabled in both old and
         * new, reuse the existing bitmap.  If logging is enabled only in the
         * new and KVM isn't using a ring buffer, allocate and initialize a
         * new bitmap.
         */
        if (change != KVM_MR_DELETE) {
                if (!(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
                        new->dirty_bitmap = NULL;
                else if (old && old->dirty_bitmap)
                        new->dirty_bitmap = old->dirty_bitmap;
                else if (kvm_use_dirty_bitmap(kvm)) {
                        r = kvm_alloc_dirty_bitmap(new);
                        if (r)
                                return r;

                        if (kvm_dirty_log_manual_protect_and_init_set(kvm))
                                bitmap_set(new->dirty_bitmap, 0, new->npages);
                }
        }

        r = kvm_arch_prepare_memory_region(kvm, old, new, change);

        /* Free the bitmap on failure if it was allocated above. */
        if (r && new && new->dirty_bitmap && (!old || !old->dirty_bitmap))
                kvm_destroy_dirty_bitmap(new);

        return r;
}

static void kvm_commit_memory_region(struct kvm *kvm,
                                     struct kvm_memory_slot *old,
                                     const struct kvm_memory_slot *new,
                                     enum kvm_mr_change change)
{
        int old_flags = old ? old->flags : 0;
        int new_flags = new ? new->flags : 0;
        /*
         * Update the total number of memslot pages before calling the arch
         * hook so that architectures can consume the result directly.
         */
        if (change == KVM_MR_DELETE)
                kvm->nr_memslot_pages -= old->npages;
        else if (change == KVM_MR_CREATE)
                kvm->nr_memslot_pages += new->npages;

        if ((old_flags ^ new_flags) & KVM_MEM_LOG_DIRTY_PAGES) {
                int change = (new_flags & KVM_MEM_LOG_DIRTY_PAGES) ? 1 : -1;
                atomic_set(&kvm->nr_memslots_dirty_logging,
                           atomic_read(&kvm->nr_memslots_dirty_logging) + change);
        }

        kvm_arch_commit_memory_region(kvm, old, new, change);

        switch (change) {
        case KVM_MR_CREATE:
                /* Nothing more to do. */
                break;
        case KVM_MR_DELETE:
                /* Free the old memslot and all its metadata. */
                kvm_free_memslot(kvm, old);
                break;
        case KVM_MR_MOVE:
        case KVM_MR_FLAGS_ONLY:
                /*
                 * Free the dirty bitmap as needed; the below check encompasses
                 * both the flags and whether a ring buffer is being used)
                 */
                if (old->dirty_bitmap && !new->dirty_bitmap)
                        kvm_destroy_dirty_bitmap(old);

                /*
                 * The final quirk.  Free the detached, old slot, but only its
                 * memory, not any metadata.  Metadata, including arch specific
                 * data, may be reused by @new.
                 */
                kfree(old);
                break;
        default:
                BUG();
        }
}

/*
 * Activate @new, which must be installed in the inactive slots by the caller,
 * by swapping the active slots and then propagating @new to @old once @old is
 * unreachable and can be safely modified.
 *
 * With NULL @old this simply adds @new to @active (while swapping the sets).
 * With NULL @new this simply removes @old from @active and frees it
 * (while also swapping the sets).
 */
static void kvm_activate_memslot(struct kvm *kvm,
                                 struct kvm_memory_slot *old,
                                 struct kvm_memory_slot *new)
{
        int as_id = kvm_memslots_get_as_id(old, new);

        kvm_swap_active_memslots(kvm, as_id);

        /* Propagate the new memslot to the now inactive memslots. */
        kvm_replace_memslot(kvm, old, new);
}

static void kvm_copy_memslot(struct kvm_memory_slot *dest,
                             const struct kvm_memory_slot *src)
{
        dest->base_gfn = src->base_gfn;
        dest->npages = src->npages;
        dest->dirty_bitmap = src->dirty_bitmap;
        dest->arch = src->arch;
        dest->userspace_addr = src->userspace_addr;
        dest->flags = src->flags;
        dest->id = src->id;
        dest->as_id = src->as_id;
}

static void kvm_invalidate_memslot(struct kvm *kvm,
                                   struct kvm_memory_slot *old,
                                   struct kvm_memory_slot *invalid_slot)
{
        /*
         * Mark the current slot INVALID.  As with all memslot modifications,
         * this must be done on an unreachable slot to avoid modifying the
         * current slot in the active tree.
         */
        kvm_copy_memslot(invalid_slot, old);
        invalid_slot->flags |= KVM_MEMSLOT_INVALID;
        kvm_replace_memslot(kvm, old, invalid_slot);

        /*
         * Activate the slot that is now marked INVALID, but don't propagate
         * the slot to the now inactive slots. The slot is either going to be
         * deleted or recreated as a new slot.
         */
        kvm_swap_active_memslots(kvm, old->as_id);

        /*
         * From this point no new shadow pages pointing to a deleted, or moved,
         * memslot will be created.  Validation of sp->gfn happens in:
         *        - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
         *        - kvm_is_visible_gfn (mmu_check_root)
         */
        kvm_arch_flush_shadow_memslot(kvm, old);
        kvm_arch_guest_memory_reclaimed(kvm);

        /* Was released by kvm_swap_active_memslots(), reacquire. */
        mutex_lock(&kvm->slots_arch_lock);

        /*
         * Copy the arch-specific field of the newly-installed slot back to the
         * old slot as the arch data could have changed between releasing
         * slots_arch_lock in kvm_swap_active_memslots() and re-acquiring the lock
         * above.  Writers are required to retrieve memslots *after* acquiring
         * slots_arch_lock, thus the active slot's data is guaranteed to be fresh.
         */
        old->arch = invalid_slot->arch;
}

static void kvm_create_memslot(struct kvm *kvm,
                               struct kvm_memory_slot *new)
{
        /* Add the new memslot to the inactive set and activate. */
        kvm_replace_memslot(kvm, NULL, new);
        kvm_activate_memslot(kvm, NULL, new);
}

static void kvm_delete_memslot(struct kvm *kvm,
                               struct kvm_memory_slot *old,
                               struct kvm_memory_slot *invalid_slot)
{
        /*
         * Remove the old memslot (in the inactive memslots) by passing NULL as
         * the "new" slot, and for the invalid version in the active slots.
         */
        kvm_replace_memslot(kvm, old, NULL);
        kvm_activate_memslot(kvm, invalid_slot, NULL);
}

static void kvm_move_memslot(struct kvm *kvm,
                             struct kvm_memory_slot *old,
                             struct kvm_memory_slot *new,
                             struct kvm_memory_slot *invalid_slot)
{
        /*
         * Replace the old memslot in the inactive slots, and then swap slots
         * and replace the current INVALID with the new as well.
         */
        kvm_replace_memslot(kvm, old, new);
        kvm_activate_memslot(kvm, invalid_slot, new);
}

static void kvm_update_flags_memslot(struct kvm *kvm,
                                     struct kvm_memory_slot *old,
                                     struct kvm_memory_slot *new)
{
        /*
         * Similar to the MOVE case, but the slot doesn't need to be zapped as
         * an intermediate step. Instead, the old memslot is simply replaced
         * with a new, updated copy in both memslot sets.
         */
        kvm_replace_memslot(kvm, old, new);
        kvm_activate_memslot(kvm, old, new);
}

static int kvm_set_memslot(struct kvm *kvm,
                           struct kvm_memory_slot *old,
                           struct kvm_memory_slot *new,
                           enum kvm_mr_change change)
{
        struct kvm_memory_slot *invalid_slot;
        int r;

        /*
         * Released in kvm_swap_active_memslots().
         *
         * Must be held from before the current memslots are copied until after
         * the new memslots are installed with rcu_assign_pointer, then
         * released before the synchronize srcu in kvm_swap_active_memslots().
         *
         * When modifying memslots outside of the slots_lock, must be held
         * before reading the pointer to the current memslots until after all
         * changes to those memslots are complete.
         *
         * These rules ensure that installing new memslots does not lose
         * changes made to the previous memslots.
         */
        mutex_lock(&kvm->slots_arch_lock);

        /*
         * Invalidate the old slot if it's being deleted or moved.  This is
         * done prior to actually deleting/moving the memslot to allow vCPUs to
         * continue running by ensuring there are no mappings or shadow pages
         * for the memslot when it is deleted/moved.  Without pre-invalidation
         * (and without a lock), a window would exist between effecting the
         * delete/move and committing the changes in arch code where KVM or a
         * guest could access a non-existent memslot.
         *
         * Modifications are done on a temporary, unreachable slot.  The old
         * slot needs to be preserved in case a later step fails and the
         * invalidation needs to be reverted.
         */
        if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
                invalid_slot = kzalloc(sizeof(*invalid_slot), GFP_KERNEL_ACCOUNT);
                if (!invalid_slot) {
                        mutex_unlock(&kvm->slots_arch_lock);
                        return -ENOMEM;
                }
                kvm_invalidate_memslot(kvm, old, invalid_slot);
        }

        r = kvm_prepare_memory_region(kvm, old, new, change);
        if (r) {
                /*
                 * For DELETE/MOVE, revert the above INVALID change.  No
                 * modifications required since the original slot was preserved
                 * in the inactive slots.  Changing the active memslots also
                 * release slots_arch_lock.
                 */
                if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
                        kvm_activate_memslot(kvm, invalid_slot, old);
                        kfree(invalid_slot);
                } else {
                        mutex_unlock(&kvm->slots_arch_lock);
                }
                return r;
        }

        /*
         * For DELETE and MOVE, the working slot is now active as the INVALID
         * version of the old slot.  MOVE is particularly special as it reuses
         * the old slot and returns a copy of the old slot (in working_slot).
         * For CREATE, there is no old slot.  For DELETE and FLAGS_ONLY, the
         * old slot is detached but otherwise preserved.
         */
        if (change == KVM_MR_CREATE)
                kvm_create_memslot(kvm, new);
        else if (change == KVM_MR_DELETE)
                kvm_delete_memslot(kvm, old, invalid_slot);
        else if (change == KVM_MR_MOVE)
                kvm_move_memslot(kvm, old, new, invalid_slot);
        else if (change == KVM_MR_FLAGS_ONLY)
                kvm_update_flags_memslot(kvm, old, new);
        else
                BUG();

        /* Free the temporary INVALID slot used for DELETE and MOVE. */
        if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
                kfree(invalid_slot);

        /*
         * No need to refresh new->arch, changes after dropping slots_arch_lock
         * will directly hit the final, active memslot.  Architectures are
         * responsible for knowing that new->arch may be stale.
         */
        kvm_commit_memory_region(kvm, old, new, change);

        return 0;
}

static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id,
                                      gfn_t start, gfn_t end)
{
        struct kvm_memslot_iter iter;

        kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) {
                if (iter.slot->id != id)
                        return true;
        }

        return false;
}

static int kvm_set_memory_region(struct kvm *kvm,
                                 const struct kvm_userspace_memory_region2 *mem)
{
        struct kvm_memory_slot *old, *new;
        struct kvm_memslots *slots;
        enum kvm_mr_change change;
        unsigned long npages;
        gfn_t base_gfn;
        int as_id, id;
        int r;

        lockdep_assert_held(&kvm->slots_lock);

        r = check_memory_region_flags(kvm, mem);
        if (r)
                return r;

        as_id = mem->slot >> 16;
        id = (u16)mem->slot;

        /* General sanity checks */
        if ((mem->memory_size & (PAGE_SIZE - 1)) ||
            (mem->memory_size != (unsigned long)mem->memory_size))
                return -EINVAL;
        if (mem->guest_phys_addr & (PAGE_SIZE - 1))
                return -EINVAL;
        /* We can read the guest memory with __xxx_user() later on. */
        if ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
            (mem->userspace_addr != untagged_addr(mem->userspace_addr)) ||
             !access_ok((void __user *)(unsigned long)mem->userspace_addr,
                        mem->memory_size))
                return -EINVAL;
        if (mem->flags & KVM_MEM_GUEST_MEMFD &&
            (mem->guest_memfd_offset & (PAGE_SIZE - 1) ||
             mem->guest_memfd_offset + mem->memory_size < mem->guest_memfd_offset))
                return -EINVAL;
        if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_MEM_SLOTS_NUM)
                return -EINVAL;
        if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
                return -EINVAL;

        /*
         * The size of userspace-defined memory regions is restricted in order
         * to play nice with dirty bitmap operations, which are indexed with an
         * "unsigned int".  KVM's internal memory regions don't support dirty
         * logging, and so are exempt.
         */
        if (id < KVM_USER_MEM_SLOTS &&
            (mem->memory_size >> PAGE_SHIFT) > KVM_MEM_MAX_NR_PAGES)
                return -EINVAL;

        slots = __kvm_memslots(kvm, as_id);

        /*
         * Note, the old memslot (and the pointer itself!) may be invalidated
         * and/or destroyed by kvm_set_memslot().
         */
        old = id_to_memslot(slots, id);

        if (!mem->memory_size) {
                if (!old || !old->npages)
                        return -EINVAL;

                if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
                        return -EIO;

                return kvm_set_memslot(kvm, old, NULL, KVM_MR_DELETE);
        }

        base_gfn = (mem->guest_phys_addr >> PAGE_SHIFT);
        npages = (mem->memory_size >> PAGE_SHIFT);

        if (!old || !old->npages) {
                change = KVM_MR_CREATE;

                /*
                 * To simplify KVM internals, the total number of pages across
                 * all memslots must fit in an unsigned long.
                 */
                if ((kvm->nr_memslot_pages + npages) < kvm->nr_memslot_pages)
                        return -EINVAL;
        } else { /* Modify an existing slot. */
                /* Private memslots are immutable, they can only be deleted. */
                if (mem->flags & KVM_MEM_GUEST_MEMFD)
                        return -EINVAL;
                if ((mem->userspace_addr != old->userspace_addr) ||
                    (npages != old->npages) ||
                    ((mem->flags ^ old->flags) & KVM_MEM_READONLY))
                        return -EINVAL;

                if (base_gfn != old->base_gfn)
                        change = KVM_MR_MOVE;
                else if (mem->flags != old->flags)
                        change = KVM_MR_FLAGS_ONLY;
                else /* Nothing to change. */
                        return 0;
        }

        if ((change == KVM_MR_CREATE || change == KVM_MR_MOVE) &&
            kvm_check_memslot_overlap(slots, id, base_gfn, base_gfn + npages))
                return -EEXIST;

        /* Allocate a slot that will persist in the memslot. */
        new = kzalloc(sizeof(*new), GFP_KERNEL_ACCOUNT);
        if (!new)
                return -ENOMEM;

        new->as_id = as_id;
        new->id = id;
        new->base_gfn = base_gfn;
        new->npages = npages;
        new->flags = mem->flags;
        new->userspace_addr = mem->userspace_addr;
        if (mem->flags & KVM_MEM_GUEST_MEMFD) {
                r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
                if (r)
                        goto out;
        }

        r = kvm_set_memslot(kvm, old, new, change);
        if (r)
                goto out_unbind;

        return 0;

out_unbind:
        if (mem->flags & KVM_MEM_GUEST_MEMFD)
                kvm_gmem_unbind(new);
out:
        kfree(new);
        return r;
}

int kvm_set_internal_memslot(struct kvm *kvm,
                             const struct kvm_userspace_memory_region2 *mem)
{
        if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS))
                return -EINVAL;

        if (WARN_ON_ONCE(mem->flags))
                return -EINVAL;

        return kvm_set_memory_region(kvm, mem);
}
EXPORT_SYMBOL_GPL(kvm_set_internal_memslot);

static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
                                          struct kvm_userspace_memory_region2 *mem)
{
        if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
                return -EINVAL;

        guard(mutex)(&kvm->slots_lock);
        return kvm_set_memory_region(kvm, mem);
}

#ifndef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
/**
 * kvm_get_dirty_log - get a snapshot of dirty pages
 * @kvm:        pointer to kvm instance
 * @log:        slot id and address to which we copy the log
 * @is_dirty:        set to '1' if any dirty pages were found
 * @memslot:        set to the associated memslot, always valid on success
 */
int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log,
                      int *is_dirty, struct kvm_memory_slot **memslot)
{
        struct kvm_memslots *slots;
        int i, as_id, id;
        unsigned long n;
        unsigned long any = 0;

        /* Dirty ring tracking may be exclusive to dirty log tracking */
        if (!kvm_use_dirty_bitmap(kvm))
                return -ENXIO;

        *memslot = NULL;
        *is_dirty = 0;

        as_id = log->slot >> 16;
        id = (u16)log->slot;
        if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
                return -EINVAL;

        slots = __kvm_memslots(kvm, as_id);
        *memslot = id_to_memslot(slots, id);
        if (!(*memslot) || !(*memslot)->dirty_bitmap)
                return -ENOENT;

        kvm_arch_sync_dirty_log(kvm, *memslot);

        n = kvm_dirty_bitmap_bytes(*memslot);

        for (i = 0; !any && i < n/sizeof(long); ++i)
                any = (*memslot)->dirty_bitmap[i];

        if (copy_to_user(log->dirty_bitmap, (*memslot)->dirty_bitmap, n))
                return -EFAULT;

        if (any)
                *is_dirty = 1;
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_dirty_log);

#else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
/**
 * kvm_get_dirty_log_protect - get a snapshot of dirty pages
 *        and reenable dirty page tracking for the corresponding pages.
 * @kvm:        pointer to kvm instance
 * @log:        slot id and address to which we copy the log
 *
 * We need to keep it in mind that VCPU threads can write to the bitmap
 * concurrently. So, to avoid losing track of dirty pages we keep the
 * following order:
 *
 *    1. Take a snapshot of the bit and clear it if needed.
 *    2. Write protect the corresponding page.
 *    3. Copy the snapshot to the userspace.
 *    4. Upon return caller flushes TLB's if needed.
 *
 * Between 2 and 4, the guest may write to the page using the remaining TLB
 * entry.  This is not a problem because the page is reported dirty using
 * the snapshot taken before and step 4 ensures that writes done after
 * exiting to userspace will be logged for the next call.
 *
 */
static int kvm_get_dirty_log_protect(struct kvm *kvm, struct kvm_dirty_log *log)
{
        struct kvm_memslots *slots;
        struct kvm_memory_slot *memslot;
        int i, as_id, id;
        unsigned long n;
        unsigned long *dirty_bitmap;
        unsigned long *dirty_bitmap_buffer;
        bool flush;

        /* Dirty ring tracking may be exclusive to dirty log tracking */
        if (!kvm_use_dirty_bitmap(kvm))
                return -ENXIO;

        as_id = log->slot >> 16;
        id = (u16)log->slot;
        if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
                return -EINVAL;

        slots = __kvm_memslots(kvm, as_id);
        memslot = id_to_memslot(slots, id);
        if (!memslot || !memslot->dirty_bitmap)
                return -ENOENT;

        dirty_bitmap = memslot->dirty_bitmap;

        kvm_arch_sync_dirty_log(kvm, memslot);

        n = kvm_dirty_bitmap_bytes(memslot);
        flush = false;
        if (kvm->manual_dirty_log_protect) {
                /*
                 * Unlike kvm_get_dirty_log, we always return false in *flush,
                 * because no flush is needed until KVM_CLEAR_DIRTY_LOG.  There
                 * is some code duplication between this function and
                 * kvm_get_dirty_log, but hopefully all architecture
                 * transition to kvm_get_dirty_log_protect and kvm_get_dirty_log
                 * can be eliminated.
                 */
                dirty_bitmap_buffer = dirty_bitmap;
        } else {
                dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
                memset(dirty_bitmap_buffer, 0, n);

                KVM_MMU_LOCK(kvm);
                for (i = 0; i < n / sizeof(long); i++) {
                        unsigned long mask;
                        gfn_t offset;

                        if (!dirty_bitmap[i])
                                continue;

                        flush = true;
                        mask = xchg(&dirty_bitmap[i], 0);
                        dirty_bitmap_buffer[i] = mask;

                        offset = i * BITS_PER_LONG;
                        kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
                                                                offset, mask);
                }
                KVM_MMU_UNLOCK(kvm);
        }

        if (flush)
                kvm_flush_remote_tlbs_memslot(kvm, memslot);

        if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
                return -EFAULT;
        return 0;
}


/**
 * kvm_vm_ioctl_get_dirty_log - get and clear the log of dirty pages in a slot
 * @kvm: kvm instance
 * @log: slot id and address to which we copy the log
 *
 * Steps 1-4 below provide general overview of dirty page logging. See
 * kvm_get_dirty_log_protect() function description for additional details.
 *
 * We call kvm_get_dirty_log_protect() to handle steps 1-3, upon return we
 * always flush the TLB (step 4) even if previous step failed  and the dirty
 * bitmap may be corrupt. Regardless of previous outcome the KVM logging API
 * does not preclude user space subsequent dirty log read. Flushing TLB ensures
 * writes will be marked dirty for next log read.
 *
 *   1. Take a snapshot of the bit and clear it if needed.
 *   2. Write protect the corresponding page.
 *   3. Copy the snapshot to the userspace.
 *   4. Flush TLB's if needed.
 */
static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
                                      struct kvm_dirty_log *log)
{
        int r;

        mutex_lock(&kvm->slots_lock);

        r = kvm_get_dirty_log_protect(kvm, log);

        mutex_unlock(&kvm->slots_lock);
        return r;
}

/**
 * kvm_clear_dirty_log_protect - clear dirty bits in the bitmap
 *        and reenable dirty page tracking for the corresponding pages.
 * @kvm:        pointer to kvm instance
 * @log:        slot id and address from which to fetch the bitmap of dirty pages
 */
static int kvm_clear_dirty_log_protect(struct kvm *kvm,
                                       struct kvm_clear_dirty_log *log)
{
        struct kvm_memslots *slots;
        struct kvm_memory_slot *memslot;
        int as_id, id;
        gfn_t offset;
        unsigned long i, n;
        unsigned long *dirty_bitmap;
        unsigned long *dirty_bitmap_buffer;
        bool flush;

        /* Dirty ring tracking may be exclusive to dirty log tracking */
        if (!kvm_use_dirty_bitmap(kvm))
                return -ENXIO;

        as_id = log->slot >> 16;
        id = (u16)log->slot;
        if (as_id >= kvm_arch_nr_memslot_as_ids(kvm) || id >= KVM_USER_MEM_SLOTS)
                return -EINVAL;

        if (log->first_page & 63)
                return -EINVAL;

        slots = __kvm_memslots(kvm, as_id);
        memslot = id_to_memslot(slots, id);
        if (!memslot || !memslot->dirty_bitmap)
                return -ENOENT;

        dirty_bitmap = memslot->dirty_bitmap;

        n = ALIGN(log->num_pages, BITS_PER_LONG) / 8;

        if (log->first_page > memslot->npages ||
            log->num_pages > memslot->npages - log->first_page ||
            (log->num_pages < memslot->npages - log->first_page && (log->num_pages & 63)))
            return -EINVAL;

        kvm_arch_sync_dirty_log(kvm, memslot);

        flush = false;
        dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
        if (copy_from_user(dirty_bitmap_buffer, log->dirty_bitmap, n))
                return -EFAULT;

        KVM_MMU_LOCK(kvm);
        for (offset = log->first_page, i = offset / BITS_PER_LONG,
                 n = DIV_ROUND_UP(log->num_pages, BITS_PER_LONG); n--;
             i++, offset += BITS_PER_LONG) {
                unsigned long mask = *dirty_bitmap_buffer++;
                atomic_long_t *p = (atomic_long_t *) &dirty_bitmap[i];
                if (!mask)
                        continue;

                mask &= atomic_long_fetch_andnot(mask, p);

                /*
                 * mask contains the bits that really have been cleared.  This
                 * never includes any bits beyond the length of the memslot (if
                 * the length is not aligned to 64 pages), therefore it is not
                 * a problem if userspace sets them in log->dirty_bitmap.
                */
                if (mask) {
                        flush = true;
                        kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
                                                                offset, mask);
                }
        }
        KVM_MMU_UNLOCK(kvm);

        if (flush)
                kvm_flush_remote_tlbs_memslot(kvm, memslot);

        return 0;
}

static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
                                        struct kvm_clear_dirty_log *log)
{
        int r;

        mutex_lock(&kvm->slots_lock);

        r = kvm_clear_dirty_log_protect(kvm, log);

        mutex_unlock(&kvm->slots_lock);
        return r;
}
#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */

#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
static u64 kvm_supported_mem_attributes(struct kvm *kvm)
{
        if (!kvm || kvm_arch_has_private_mem(kvm))
                return KVM_MEMORY_ATTRIBUTE_PRIVATE;

        return 0;
}

/*
 * Returns true if _all_ gfns in the range [@start, @end) have attributes
 * such that the bits in @mask match @attrs.
 */
bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
                                     unsigned long mask, unsigned long attrs)
{
        XA_STATE(xas, &kvm->mem_attr_array, start);
        unsigned long index;
        void *entry;

        mask &= kvm_supported_mem_attributes(kvm);
        if (attrs & ~mask)
                return false;

        if (end == start + 1)
                return (kvm_get_memory_attributes(kvm, start) & mask) == attrs;

        guard(rcu)();
        if (!attrs)
                return !xas_find(&xas, end - 1);

        for (index = start; index < end; index++) {
                do {
                        entry = xas_next(&xas);
                } while (xas_retry(&xas, entry));

                if (xas.xa_index != index ||
                    (xa_to_value(entry) & mask) != attrs)
                        return false;
        }

        return true;
}

static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
                                                 struct kvm_mmu_notifier_range *range)
{
        struct kvm_gfn_range gfn_range;
        struct kvm_memory_slot *slot;
        struct kvm_memslots *slots;
        struct kvm_memslot_iter iter;
        bool found_memslot = false;
        bool ret = false;
        int i;

        gfn_range.arg = range->arg;
        gfn_range.may_block = range->may_block;

        /*
         * If/when KVM supports more attributes beyond private .vs shared, this
         * _could_ set KVM_FILTER_{SHARED,PRIVATE} appropriately if the entire target
         * range already has the desired private vs. shared state (it's unclear
         * if that is a net win).  For now, KVM reaches this point if and only
         * if the private flag is being toggled, i.e. all mappings are in play.
         */

        for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
                slots = __kvm_memslots(kvm, i);

                kvm_for_each_memslot_in_gfn_range(&iter, slots, range->start, range->end) {
                        slot = iter.slot;
                        gfn_range.slot = slot;

                        gfn_range.start = max(range->start, slot->base_gfn);
                        gfn_range.end = min(range->end, slot->base_gfn + slot->npages);
                        if (gfn_range.start >= gfn_range.end)
                                continue;

                        if (!found_memslot) {
                                found_memslot = true;
                                KVM_MMU_LOCK(kvm);
                                if (!IS_KVM_NULL_FN(range->on_lock))
                                        range->on_lock(kvm);
                        }

                        ret |= range->handler(kvm, &gfn_range);
                }
        }

        if (range->flush_on_ret && ret)
                kvm_flush_remote_tlbs(kvm);

        if (found_memslot)
                KVM_MMU_UNLOCK(kvm);
}

static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
                                          struct kvm_gfn_range *range)
{
        /*
         * Unconditionally add the range to the invalidation set, regardless of
         * whether or not the arch callback actually needs to zap SPTEs.  E.g.
         * if KVM supports RWX attributes in the future and the attributes are
         * going from R=>RW, zapping isn't strictly necessary.  Unconditionally
         * adding the range allows KVM to require that MMU invalidations add at
         * least one range between begin() and end(), e.g. allows KVM to detect
         * bugs where the add() is missed.  Relaxing the rule *might* be safe,
         * but it's not obvious that allowing new mappings while the attributes
         * are in flux is desirable or worth the complexity.
         */
        kvm_mmu_invalidate_range_add(kvm, range->start, range->end);

        return kvm_arch_pre_set_memory_attributes(kvm, range);
}

/* Set @attributes for the gfn range [@start, @end). */
static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
                                     unsigned long attributes)
{
        struct kvm_mmu_notifier_range pre_set_range = {
                .start = start,
                .end = end,
                .arg.attributes = attributes,
                .handler = kvm_pre_set_memory_attributes,
                .on_lock = kvm_mmu_invalidate_begin,
                .flush_on_ret = true,
                .may_block = true,
        };
        struct kvm_mmu_notifier_range post_set_range = {
                .start = start,
                .end = end,
                .arg.attributes = attributes,
                .handler = kvm_arch_post_set_memory_attributes,
                .on_lock = kvm_mmu_invalidate_end,
                .may_block = true,
        };
        unsigned long i;
        void *entry;
        int r = 0;

        entry = attributes ? xa_mk_value(attributes) : NULL;

        mutex_lock(&kvm->slots_lock);

        /* Nothing to do if the entire range as the desired attributes. */
        if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes))
                goto out_unlock;

        /*
         * Reserve memory ahead of time to avoid having to deal with failures
         * partway through setting the new attributes.
         */
        for (i = start; i < end; i++) {
                r = xa_reserve(&kvm->mem_attr_array, i, GFP_KERNEL_ACCOUNT);
                if (r)
                        goto out_unlock;
        }

        kvm_handle_gfn_range(kvm, &pre_set_range);

        for (i = start; i < end; i++) {
                r = xa_err(xa_store(&kvm->mem_attr_array, i, entry,
                                    GFP_KERNEL_ACCOUNT));
                KVM_BUG_ON(r, kvm);
        }

        kvm_handle_gfn_range(kvm, &post_set_range);

out_unlock:
        mutex_unlock(&kvm->slots_lock);

        return r;
}
static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
                                           struct kvm_memory_attributes *attrs)
{
        gfn_t start, end;

        /* flags is currently not used. */
        if (attrs->flags)
                return -EINVAL;
        if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
                return -EINVAL;
        if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
                return -EINVAL;
        if (!PAGE_ALIGNED(attrs->address) || !PAGE_ALIGNED(attrs->size))
                return -EINVAL;

        start = attrs->address >> PAGE_SHIFT;
        end = (attrs->address + attrs->size) >> PAGE_SHIFT;

        /*
         * xarray tracks data using "unsigned long", and as a result so does
         * KVM.  For simplicity, supports generic attributes only on 64-bit
         * architectures.
         */
        BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));

        return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
}
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */

struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
        return __gfn_to_memslot(kvm_memslots(kvm), gfn);
}
EXPORT_SYMBOL_GPL(gfn_to_memslot);

struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
{
        struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
        u64 gen = slots->generation;
        struct kvm_memory_slot *slot;

        /*
         * This also protects against using a memslot from a different address space,
         * since different address spaces have different generation numbers.
         */
        if (unlikely(gen != vcpu->last_used_slot_gen)) {
                vcpu->last_used_slot = NULL;
                vcpu->last_used_slot_gen = gen;
        }

        slot = try_get_memslot(vcpu->last_used_slot, gfn);
        if (slot)
                return slot;

        /*
         * Fall back to searching all memslots. We purposely use
         * search_memslots() instead of __gfn_to_memslot() to avoid
         * thrashing the VM-wide last_used_slot in kvm_memslots.
         */
        slot = search_memslots(slots, gfn, false);
        if (slot) {
                vcpu->last_used_slot = slot;
                return slot;
        }

        return NULL;
}

bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
{
        struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);

        return kvm_is_visible_memslot(memslot);
}
EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);

bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn)
{
        struct kvm_memory_slot *memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

        return kvm_is_visible_memslot(memslot);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn);

unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn)
{
        struct vm_area_struct *vma;
        unsigned long addr, size;

        size = PAGE_SIZE;

        addr = kvm_vcpu_gfn_to_hva_prot(vcpu, gfn, NULL);
        if (kvm_is_error_hva(addr))
                return PAGE_SIZE;

        mmap_read_lock(current->mm);
        vma = find_vma(current->mm, addr);
        if (!vma)
                goto out;

        size = vma_kernel_pagesize(vma);

out:
        mmap_read_unlock(current->mm);

        return size;
}

static bool memslot_is_readonly(const struct kvm_memory_slot *slot)
{
        return slot->flags & KVM_MEM_READONLY;
}

static unsigned long __gfn_to_hva_many(const struct kvm_memory_slot *slot, gfn_t gfn,
                                       gfn_t *nr_pages, bool write)
{
        if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
                return KVM_HVA_ERR_BAD;

        if (memslot_is_readonly(slot) && write)
                return KVM_HVA_ERR_RO_BAD;

        if (nr_pages)
                *nr_pages = slot->npages - (gfn - slot->base_gfn);

        return __gfn_to_hva_memslot(slot, gfn);
}

static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
                                     gfn_t *nr_pages)
{
        return __gfn_to_hva_many(slot, gfn, nr_pages, true);
}

unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
                                        gfn_t gfn)
{
        return gfn_to_hva_many(slot, gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);

unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
{
        return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
}
EXPORT_SYMBOL_GPL(gfn_to_hva);

unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
{
        return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);

/*
 * Return the hva of a @gfn and the R/W attribute if possible.
 *
 * @slot: the kvm_memory_slot which contains @gfn
 * @gfn: the gfn to be translated
 * @writable: used to return the read/write attribute of the @slot if the hva
 * is valid and @writable is not NULL
 */
unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
                                      gfn_t gfn, bool *writable)
{
        unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);

        if (!kvm_is_error_hva(hva) && writable)
                *writable = !memslot_is_readonly(slot);

        return hva;
}

unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
{
        struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

        return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
{
        struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

        return gfn_to_hva_memslot_prot(slot, gfn, writable);
}

static bool kvm_is_ad_tracked_page(struct page *page)
{
        /*
         * Per page-flags.h, pages tagged PG_reserved "should in general not be
         * touched (e.g. set dirty) except by its owner".
         */
        return !PageReserved(page);
}

static void kvm_set_page_dirty(struct page *page)
{
        if (kvm_is_ad_tracked_page(page))
                SetPageDirty(page);
}

static void kvm_set_page_accessed(struct page *page)
{
        if (kvm_is_ad_tracked_page(page))
                mark_page_accessed(page);
}

void kvm_release_page_clean(struct page *page)
{
        if (!page)
                return;

        kvm_set_page_accessed(page);
        put_page(page);
}
EXPORT_SYMBOL_GPL(kvm_release_page_clean);

void kvm_release_page_dirty(struct page *page)
{
        if (!page)
                return;

        kvm_set_page_dirty(page);
        kvm_release_page_clean(page);
}
EXPORT_SYMBOL_GPL(kvm_release_page_dirty);

static kvm_pfn_t kvm_resolve_pfn(struct kvm_follow_pfn *kfp, struct page *page,
                                 struct follow_pfnmap_args *map, bool writable)
{
        kvm_pfn_t pfn;

        WARN_ON_ONCE(!!page == !!map);

        if (kfp->map_writable)
                *kfp->map_writable = writable;

        if (map)
                pfn = map->pfn;
        else
                pfn = page_to_pfn(page);

        *kfp->refcounted_page = page;

        return pfn;
}

/*
 * The fast path to get the writable pfn which will be stored in @pfn,
 * true indicates success, otherwise false is returned.
 */
static bool hva_to_pfn_fast(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
{
        struct page *page;
        bool r;

        /*
         * Try the fast-only path when the caller wants to pin/get the page for
         * writing.  If the caller only wants to read the page, KVM must go
         * down the full, slow path in order to avoid racing an operation that
         * breaks Copy-on-Write (CoW), e.g. so that KVM doesn't end up pointing
         * at the old, read-only page while mm/ points at a new, writable page.
         */
        if (!((kfp->flags & FOLL_WRITE) || kfp->map_writable))
                return false;

        if (kfp->pin)
                r = pin_user_pages_fast(kfp->hva, 1, FOLL_WRITE, &page) == 1;
        else
                r = get_user_page_fast_only(kfp->hva, FOLL_WRITE, &page);

        if (r) {
                *pfn = kvm_resolve_pfn(kfp, page, NULL, true);
                return true;
        }

        return false;
}

/*
 * The slow path to get the pfn of the specified host virtual address,
 * 1 indicates success, -errno is returned if error is detected.
 */
static int hva_to_pfn_slow(struct kvm_follow_pfn *kfp, kvm_pfn_t *pfn)
{
        /*
         * When a VCPU accesses a page that is not mapped into the secondary
         * MMU, we lookup the page using GUP to map it, so the guest VCPU can
         * make progress. We always want to honor NUMA hinting faults in that
         * case, because GUP usage corresponds to memory accesses from the VCPU.
         * Otherwise, we'd not trigger NUMA hinting faults once a page is
         * mapped into the secondary MMU and gets accessed by a VCPU.
         *
         * Note that get_user_page_fast_only() and FOLL_WRITE for now
         * implicitly honor NUMA hinting faults and don't need this flag.
         */
        unsigned int flags = FOLL_HWPOISON | FOLL_HONOR_NUMA_FAULT | kfp->flags;
        struct page *page, *wpage;
        int npages;

        if (kfp->pin)
                npages = pin_user_pages_unlocked(kfp->hva, 1, &page, flags);
        else
                npages = get_user_pages_unlocked(kfp->hva, 1, &page, flags);
        if (npages != 1)
                return npages;

        /*
         * Pinning is mutually exclusive with opportunistically mapping a read
         * fault as writable, as KVM should never pin pages when mapping memory
         * into the guest (pinning is only for direct accesses from KVM).
         */
        if (WARN_ON_ONCE(kfp->map_writable && kfp->pin))
                goto out;

        /* map read fault as writable if possible */
        if (!(flags & FOLL_WRITE) && kfp->map_writable &&
            get_user_page_fast_only(kfp->hva, FOLL_WRITE, &wpage)) {
                put_page(page);
                page = wpage;
                flags |= FOLL_WRITE;
        }

out:
        *pfn = kvm_resolve_pfn(kfp, page, NULL, flags & FOLL_WRITE);
        return npages;
}

static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
{
        if (unlikely(!(vma->vm_flags & VM_READ)))
                return false;

        if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
                return false;

        return true;
}

static int hva_to_pfn_remapped(struct vm_area_struct *vma,
                               struct kvm_follow_pfn *kfp, kvm_pfn_t *p_pfn)
{
        struct follow_pfnmap_args args = { .vma = vma, .address = kfp->hva };
        bool write_fault = kfp->flags & FOLL_WRITE;
        int r;

        /*
         * Remapped memory cannot be pinned in any meaningful sense.  Bail if
         * the caller wants to pin the page, i.e. access the page outside of
         * MMU notifier protection, and unsafe umappings are disallowed.
         */
        if (kfp->pin && !allow_unsafe_mappings)
                return -EINVAL;

        r = follow_pfnmap_start(&args);
        if (r) {
                /*
                 * get_user_pages fails for VM_IO and VM_PFNMAP vmas and does
                 * not call the fault handler, so do it here.
                 */
                bool unlocked = false;
                r = fixup_user_fault(current->mm, kfp->hva,
                                     (write_fault ? FAULT_FLAG_WRITE : 0),
                                     &unlocked);
                if (unlocked)
                        return -EAGAIN;
                if (r)
                        return r;

                r = follow_pfnmap_start(&args);
                if (r)
                        return r;
        }

        if (write_fault && !args.writable) {
                *p_pfn = KVM_PFN_ERR_RO_FAULT;
                goto out;
        }

        *p_pfn = kvm_resolve_pfn(kfp, NULL, &args, args.writable);
out:
        follow_pfnmap_end(&args);
        return r;
}

kvm_pfn_t hva_to_pfn(struct kvm_follow_pfn *kfp)
{
        struct vm_area_struct *vma;
        kvm_pfn_t pfn;
        int npages, r;

        might_sleep();

        if (WARN_ON_ONCE(!kfp->refcounted_page))
                return KVM_PFN_ERR_FAULT;

        if (hva_to_pfn_fast(kfp, &pfn))
                return pfn;

        npages = hva_to_pfn_slow(kfp, &pfn);
        if (npages == 1)
                return pfn;
        if (npages == -EINTR || npages == -EAGAIN)
                return KVM_PFN_ERR_SIGPENDING;
        if (npages == -EHWPOISON)
                return KVM_PFN_ERR_HWPOISON;

        mmap_read_lock(current->mm);
retry:
        vma = vma_lookup(current->mm, kfp->hva);

        if (vma == NULL)
                pfn = KVM_PFN_ERR_FAULT;
        else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
                r = hva_to_pfn_remapped(vma, kfp, &pfn);
                if (r == -EAGAIN)
                        goto retry;
                if (r < 0)
                        pfn = KVM_PFN_ERR_FAULT;
        } else {
                if ((kfp->flags & FOLL_NOWAIT) &&
                    vma_is_valid(vma, kfp->flags & FOLL_WRITE))
                        pfn = KVM_PFN_ERR_NEEDS_IO;
                else
                        pfn = KVM_PFN_ERR_FAULT;
        }
        mmap_read_unlock(current->mm);
        return pfn;
}

static kvm_pfn_t kvm_follow_pfn(struct kvm_follow_pfn *kfp)
{
        kfp->hva = __gfn_to_hva_many(kfp->slot, kfp->gfn, NULL,
                                     kfp->flags & FOLL_WRITE);

        if (kfp->hva == KVM_HVA_ERR_RO_BAD)
                return KVM_PFN_ERR_RO_FAULT;

        if (kvm_is_error_hva(kfp->hva))
                return KVM_PFN_NOSLOT;

        if (memslot_is_readonly(kfp->slot) && kfp->map_writable) {
                *kfp->map_writable = false;
                kfp->map_writable = NULL;
        }

        return hva_to_pfn(kfp);
}

kvm_pfn_t __kvm_faultin_pfn(const struct kvm_memory_slot *slot, gfn_t gfn,
                            unsigned int foll, bool *writable,
                            struct page **refcounted_page)
{
        struct kvm_follow_pfn kfp = {
                .slot = slot,
                .gfn = gfn,
                .flags = foll,
                .map_writable = writable,
                .refcounted_page = refcounted_page,
        };

        if (WARN_ON_ONCE(!writable || !refcounted_page))
                return KVM_PFN_ERR_FAULT;

        *writable = false;
        *refcounted_page = NULL;

        return kvm_follow_pfn(&kfp);
}
EXPORT_SYMBOL_GPL(__kvm_faultin_pfn);

int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn,
                       struct page **pages, int nr_pages)
{
        unsigned long addr;
        gfn_t entry = 0;

        addr = gfn_to_hva_many(slot, gfn, &entry);
        if (kvm_is_error_hva(addr))
                return -1;

        if (entry < nr_pages)
                return 0;

        return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages);
}
EXPORT_SYMBOL_GPL(kvm_prefetch_pages);

/*
 * Don't use this API unless you are absolutely, positively certain that KVM
 * needs to get a struct page, e.g. to pin the page for firmware DMA.
 *
 * FIXME: Users of this API likely need to FOLL_PIN the page, not just elevate
 *          its refcount.
 */
struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn, bool write)
{
        struct page *refcounted_page = NULL;
        struct kvm_follow_pfn kfp = {
                .slot = gfn_to_memslot(kvm, gfn),
                .gfn = gfn,
                .flags = write ? FOLL_WRITE : 0,
                .refcounted_page = &refcounted_page,
        };

        (void)kvm_follow_pfn(&kfp);
        return refcounted_page;
}
EXPORT_SYMBOL_GPL(__gfn_to_page);

int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
                   bool writable)
{
        struct kvm_follow_pfn kfp = {
                .slot = gfn_to_memslot(vcpu->kvm, gfn),
                .gfn = gfn,
                .flags = writable ? FOLL_WRITE : 0,
                .refcounted_page = &map->pinned_page,
                .pin = true,
        };

        map->pinned_page = NULL;
        map->page = NULL;
        map->hva = NULL;
        map->gfn = gfn;
        map->writable = writable;

        map->pfn = kvm_follow_pfn(&kfp);
        if (is_error_noslot_pfn(map->pfn))
                return -EINVAL;

        if (pfn_valid(map->pfn)) {
                map->page = pfn_to_page(map->pfn);
                map->hva = kmap(map->page);
#ifdef CONFIG_HAS_IOMEM
        } else {
                map->hva = memremap(pfn_to_hpa(map->pfn), PAGE_SIZE, MEMREMAP_WB);
#endif
        }

        return map->hva ? 0 : -EFAULT;
}
EXPORT_SYMBOL_GPL(__kvm_vcpu_map);

void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map)
{
        if (!map->hva)
                return;

        if (map->page)
                kunmap(map->page);
#ifdef CONFIG_HAS_IOMEM
        else
                memunmap(map->hva);
#endif

        if (map->writable)
                kvm_vcpu_mark_page_dirty(vcpu, map->gfn);

        if (map->pinned_page) {
                if (map->writable)
                        kvm_set_page_dirty(map->pinned_page);
                kvm_set_page_accessed(map->pinned_page);
                unpin_user_page(map->pinned_page);
        }

        map->hva = NULL;
        map->page = NULL;
        map->pinned_page = NULL;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_unmap);

static int next_segment(unsigned long len, int offset)
{
        if (len > PAGE_SIZE - offset)
                return PAGE_SIZE - offset;
        else
                return len;
}

/* Copy @len bytes from guest memory at '(@gfn * PAGE_SIZE) + @offset' to @data */
static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
                                 void *data, int offset, int len)
{
        int r;
        unsigned long addr;

        if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
                return -EFAULT;

        addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
        if (kvm_is_error_hva(addr))
                return -EFAULT;
        r = __copy_from_user(data, (void __user *)addr + offset, len);
        if (r)
                return -EFAULT;
        return 0;
}

int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
                        int len)
{
        struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

        return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page);

int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
                             int offset, int len)
{
        struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

        return __kvm_read_guest_page(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);

int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
{
        gfn_t gfn = gpa >> PAGE_SHIFT;
        int seg;
        int offset = offset_in_page(gpa);
        int ret;

        while ((seg = next_segment(len, offset)) != 0) {
                ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
                if (ret < 0)
                        return ret;
                offset = 0;
                len -= seg;
                data += seg;
                ++gfn;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest);

int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
{
        gfn_t gfn = gpa >> PAGE_SHIFT;
        int seg;
        int offset = offset_in_page(gpa);
        int ret;

        while ((seg = next_segment(len, offset)) != 0) {
                ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
                if (ret < 0)
                        return ret;
                offset = 0;
                len -= seg;
                data += seg;
                ++gfn;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);

static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
                                   void *data, int offset, unsigned long len)
{
        int r;
        unsigned long addr;

        if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
                return -EFAULT;

        addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
        if (kvm_is_error_hva(addr))
                return -EFAULT;
        pagefault_disable();
        r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
        pagefault_enable();
        if (r)
                return -EFAULT;
        return 0;
}

int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
                               void *data, unsigned long len)
{
        gfn_t gfn = gpa >> PAGE_SHIFT;
        struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
        int offset = offset_in_page(gpa);

        return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);

/* Copy @len bytes from @data into guest memory at '(@gfn * PAGE_SIZE) + @offset' */
static int __kvm_write_guest_page(struct kvm *kvm,
                                  struct kvm_memory_slot *memslot, gfn_t gfn,
                                  const void *data, int offset, int len)
{
        int r;
        unsigned long addr;

        if (WARN_ON_ONCE(offset + len > PAGE_SIZE))
                return -EFAULT;

        addr = gfn_to_hva_memslot(memslot, gfn);
        if (kvm_is_error_hva(addr))
                return -EFAULT;
        r = __copy_to_user((void __user *)addr + offset, data, len);
        if (r)
                return -EFAULT;
        mark_page_dirty_in_slot(kvm, memslot, gfn);
        return 0;
}

int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
                         const void *data, int offset, int len)
{
        struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);

        return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_write_guest_page);

int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
                              const void *data, int offset, int len)
{
        struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);

        return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);

int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
                    unsigned long len)
{
        gfn_t gfn = gpa >> PAGE_SHIFT;
        int seg;
        int offset = offset_in_page(gpa);
        int ret;

        while ((seg = next_segment(len, offset)) != 0) {
                ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
                if (ret < 0)
                        return ret;
                offset = 0;
                len -= seg;
                data += seg;
                ++gfn;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest);

int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
                         unsigned long len)
{
        gfn_t gfn = gpa >> PAGE_SHIFT;
        int seg;
        int offset = offset_in_page(gpa);
        int ret;

        while ((seg = next_segment(len, offset)) != 0) {
                ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
                if (ret < 0)
                        return ret;
                offset = 0;
                len -= seg;
                data += seg;
                ++gfn;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);

static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots,
                                       struct gfn_to_hva_cache *ghc,
                                       gpa_t gpa, unsigned long len)
{
        int offset = offset_in_page(gpa);
        gfn_t start_gfn = gpa >> PAGE_SHIFT;
        gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
        gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
        gfn_t nr_pages_avail;

        /* Update ghc->generation before performing any error checks. */
        ghc->generation = slots->generation;

        if (start_gfn > end_gfn) {
                ghc->hva = KVM_HVA_ERR_BAD;
                return -EINVAL;
        }

        /*
         * If the requested region crosses two memslots, we still
         * verify that the entire region is valid here.
         */
        for ( ; start_gfn <= end_gfn; start_gfn += nr_pages_avail) {
                ghc->memslot = __gfn_to_memslot(slots, start_gfn);
                ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
                                           &nr_pages_avail);
                if (kvm_is_error_hva(ghc->hva))
                        return -EFAULT;
        }

        /* Use the slow path for cross page reads and writes. */
        if (nr_pages_needed == 1)
                ghc->hva += offset;
        else
                ghc->memslot = NULL;

        ghc->gpa = gpa;
        ghc->len = len;
        return 0;
}

int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
                              gpa_t gpa, unsigned long len)
{
        struct kvm_memslots *slots = kvm_memslots(kvm);
        return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len);
}
EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);

int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
                                  void *data, unsigned int offset,
                                  unsigned long len)
{
        struct kvm_memslots *slots = kvm_memslots(kvm);
        int r;
        gpa_t gpa = ghc->gpa + offset;

        if (WARN_ON_ONCE(len + offset > ghc->len))
                return -EINVAL;

        if (slots->generation != ghc->generation) {
                if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
                        return -EFAULT;
        }

        if (kvm_is_error_hva(ghc->hva))
                return -EFAULT;

        if (unlikely(!ghc->memslot))
                return kvm_write_guest(kvm, gpa, data, len);

        r = __copy_to_user((void __user *)ghc->hva + offset, data, len);
        if (r)
                return -EFAULT;
        mark_page_dirty_in_slot(kvm, ghc->memslot, gpa >> PAGE_SHIFT);

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached);

int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
                           void *data, unsigned long len)
{
        return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len);
}
EXPORT_SYMBOL_GPL(kvm_write_guest_cached);

int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
                                 void *data, unsigned int offset,
                                 unsigned long len)
{
        struct kvm_memslots *slots = kvm_memslots(kvm);
        int r;
        gpa_t gpa = ghc->gpa + offset;

        if (WARN_ON_ONCE(len + offset > ghc->len))
                return -EINVAL;

        if (slots->generation != ghc->generation) {
                if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len))
                        return -EFAULT;
        }

        if (kvm_is_error_hva(ghc->hva))
                return -EFAULT;

        if (unlikely(!ghc->memslot))
                return kvm_read_guest(kvm, gpa, data, len);

        r = __copy_from_user(data, (void __user *)ghc->hva + offset, len);
        if (r)
                return -EFAULT;

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached);

int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
                          void *data, unsigned long len)
{
        return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_cached);

int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
{
        const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
        gfn_t gfn = gpa >> PAGE_SHIFT;
        int seg;
        int offset = offset_in_page(gpa);
        int ret;

        while ((seg = next_segment(len, offset)) != 0) {
                ret = kvm_write_guest_page(kvm, gfn, zero_page, offset, seg);
                if (ret < 0)
                        return ret;
                offset = 0;
                len -= seg;
                ++gfn;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_clear_guest);

void mark_page_dirty_in_slot(struct kvm *kvm,
                             const struct kvm_memory_slot *memslot,
                              gfn_t gfn)
{
        struct kvm_vcpu *vcpu = kvm_get_running_vcpu();

#ifdef CONFIG_HAVE_KVM_DIRTY_RING
        if (WARN_ON_ONCE(vcpu && vcpu->kvm != kvm))
                return;

        WARN_ON_ONCE(!vcpu && !kvm_arch_allow_write_without_running_vcpu(kvm));
#endif

        if (memslot && kvm_slot_dirty_track_enabled(memslot)) {
                unsigned long rel_gfn = gfn - memslot->base_gfn;
                u32 slot = (memslot->as_id << 16) | memslot->id;

                if (kvm->dirty_ring_size && vcpu)
                        kvm_dirty_ring_push(vcpu, slot, rel_gfn);
                else if (memslot->dirty_bitmap)
                        set_bit_le(rel_gfn, memslot->dirty_bitmap);
        }
}
EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot);

void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
{
        struct kvm_memory_slot *memslot;

        memslot = gfn_to_memslot(kvm, gfn);
        mark_page_dirty_in_slot(kvm, memslot, gfn);
}
EXPORT_SYMBOL_GPL(mark_page_dirty);

void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
{
        struct kvm_memory_slot *memslot;

        memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
        mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);

void kvm_sigset_activate(struct kvm_vcpu *vcpu)
{
        if (!vcpu->sigset_active)
                return;

        /*
         * This does a lockless modification of ->real_blocked, which is fine
         * because, only current can change ->real_blocked and all readers of
         * ->real_blocked don't care as long ->real_blocked is always a subset
         * of ->blocked.
         */
        sigprocmask(SIG_SETMASK, &vcpu->sigset, &current->real_blocked);
}

void kvm_sigset_deactivate(struct kvm_vcpu *vcpu)
{
        if (!vcpu->sigset_active)
                return;

        sigprocmask(SIG_SETMASK, &current->real_blocked, NULL);
        sigemptyset(&current->real_blocked);
}

static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
{
        unsigned int old, val, grow, grow_start;

        old = val = vcpu->halt_poll_ns;
        grow_start = READ_ONCE(halt_poll_ns_grow_start);
        grow = READ_ONCE(halt_poll_ns_grow);
        if (!grow)
                goto out;

        val *= grow;
        if (val < grow_start)
                val = grow_start;

        vcpu->halt_poll_ns = val;
out:
        trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
}

static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
{
        unsigned int old, val, shrink, grow_start;

        old = val = vcpu->halt_poll_ns;
        shrink = READ_ONCE(halt_poll_ns_shrink);
        grow_start = READ_ONCE(halt_poll_ns_grow_start);
        if (shrink == 0)
                val = 0;
        else
                val /= shrink;

        if (val < grow_start)
                val = 0;

        vcpu->halt_poll_ns = val;
        trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
}

static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
{
        int ret = -EINTR;
        int idx = srcu_read_lock(&vcpu->kvm->srcu);

        if (kvm_arch_vcpu_runnable(vcpu))
                goto out;
        if (kvm_cpu_has_pending_timer(vcpu))
                goto out;
        if (signal_pending(current))
                goto out;
        if (kvm_check_request(KVM_REQ_UNBLOCK, vcpu))
                goto out;

        ret = 0;
out:
        srcu_read_unlock(&vcpu->kvm->srcu, idx);
        return ret;
}

/*
 * Block the vCPU until the vCPU is runnable, an event arrives, or a signal is
 * pending.  This is mostly used when halting a vCPU, but may also be used
 * directly for other vCPU non-runnable states, e.g. x86's Wait-For-SIPI.
 */
bool kvm_vcpu_block(struct kvm_vcpu *vcpu)
{
        struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
        bool waited = false;

        vcpu->stat.generic.blocking = 1;

        preempt_disable();
        kvm_arch_vcpu_blocking(vcpu);
        prepare_to_rcuwait(wait);
        preempt_enable();

        for (;;) {
                set_current_state(TASK_INTERRUPTIBLE);

                if (kvm_vcpu_check_block(vcpu) < 0)
                        break;

                waited = true;
                schedule();
        }

        preempt_disable();
        finish_rcuwait(wait);
        kvm_arch_vcpu_unblocking(vcpu);
        preempt_enable();

        vcpu->stat.generic.blocking = 0;

        return waited;
}

static inline void update_halt_poll_stats(struct kvm_vcpu *vcpu, ktime_t start,
                                          ktime_t end, bool success)
{
        struct kvm_vcpu_stat_generic *stats = &vcpu->stat.generic;
        u64 poll_ns = ktime_to_ns(ktime_sub(end, start));

        ++vcpu->stat.generic.halt_attempted_poll;

        if (success) {
                ++vcpu->stat.generic.halt_successful_poll;

                if (!vcpu_valid_wakeup(vcpu))
                        ++vcpu->stat.generic.halt_poll_invalid;

                stats->halt_poll_success_ns += poll_ns;
                KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_success_hist, poll_ns);
        } else {
                stats->halt_poll_fail_ns += poll_ns;
                KVM_STATS_LOG_HIST_UPDATE(stats->halt_poll_fail_hist, poll_ns);
        }
}

static unsigned int kvm_vcpu_max_halt_poll_ns(struct kvm_vcpu *vcpu)
{
        struct kvm *kvm = vcpu->kvm;

        if (kvm->override_halt_poll_ns) {
                /*
                 * Ensure kvm->max_halt_poll_ns is not read before
                 * kvm->override_halt_poll_ns.
                 *
                 * Pairs with the smp_wmb() when enabling KVM_CAP_HALT_POLL.
                 */
                smp_rmb();
                return READ_ONCE(kvm->max_halt_poll_ns);
        }

        return READ_ONCE(halt_poll_ns);
}

/*
 * Emulate a vCPU halt condition, e.g. HLT on x86, WFI on arm, etc...  If halt
 * polling is enabled, busy wait for a short time before blocking to avoid the
 * expensive block+unblock sequence if a wake event arrives soon after the vCPU
 * is halted.
 */
void kvm_vcpu_halt(struct kvm_vcpu *vcpu)
{
        unsigned int max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);
        bool halt_poll_allowed = !kvm_arch_no_poll(vcpu);
        ktime_t start, cur, poll_end;
        bool waited = false;
        bool do_halt_poll;
        u64 halt_ns;

        if (vcpu->halt_poll_ns > max_halt_poll_ns)
                vcpu->halt_poll_ns = max_halt_poll_ns;

        do_halt_poll = halt_poll_allowed && vcpu->halt_poll_ns;

        start = cur = poll_end = ktime_get();
        if (do_halt_poll) {
                ktime_t stop = ktime_add_ns(start, vcpu->halt_poll_ns);

                do {
                        if (kvm_vcpu_check_block(vcpu) < 0)
                                goto out;
                        cpu_relax();
                        poll_end = cur = ktime_get();
                } while (kvm_vcpu_can_poll(cur, stop));
        }

        waited = kvm_vcpu_block(vcpu);

        cur = ktime_get();
        if (waited) {
                vcpu->stat.generic.halt_wait_ns +=
                        ktime_to_ns(cur) - ktime_to_ns(poll_end);
                KVM_STATS_LOG_HIST_UPDATE(vcpu->stat.generic.halt_wait_hist,
                                ktime_to_ns(cur) - ktime_to_ns(poll_end));
        }
out:
        /* The total time the vCPU was "halted", including polling time. */
        halt_ns = ktime_to_ns(cur) - ktime_to_ns(start);

        /*
         * Note, halt-polling is considered successful so long as the vCPU was
         * never actually scheduled out, i.e. even if the wake event arrived
         * after of the halt-polling loop itself, but before the full wait.
         */
        if (do_halt_poll)
                update_halt_poll_stats(vcpu, start, poll_end, !waited);

        if (halt_poll_allowed) {
                /* Recompute the max halt poll time in case it changed. */
                max_halt_poll_ns = kvm_vcpu_max_halt_poll_ns(vcpu);

                if (!vcpu_valid_wakeup(vcpu)) {
                        shrink_halt_poll_ns(vcpu);
                } else if (max_halt_poll_ns) {
                        if (halt_ns <= vcpu->halt_poll_ns)
                                ;
                        /* we had a long block, shrink polling */
                        else if (vcpu->halt_poll_ns &&
                                 halt_ns > max_halt_poll_ns)
                                shrink_halt_poll_ns(vcpu);
                        /* we had a short halt and our poll time is too small */
                        else if (vcpu->halt_poll_ns < max_halt_poll_ns &&
                                 halt_ns < max_halt_poll_ns)
                                grow_halt_poll_ns(vcpu);
                } else {
                        vcpu->halt_poll_ns = 0;
                }
        }

        trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu));
}
EXPORT_SYMBOL_GPL(kvm_vcpu_halt);

bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
{
        if (__kvm_vcpu_wake_up(vcpu)) {
                WRITE_ONCE(vcpu->ready, true);
                ++vcpu->stat.generic.halt_wakeup;
                return true;
        }

        return false;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up);

#ifndef CONFIG_S390
/*
 * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
 */
void __kvm_vcpu_kick(struct kvm_vcpu *vcpu, bool wait)
{
        int me, cpu;

        if (kvm_vcpu_wake_up(vcpu))
                return;

        me = get_cpu();
        /*
         * The only state change done outside the vcpu mutex is IN_GUEST_MODE
         * to EXITING_GUEST_MODE.  Therefore the moderately expensive "should
         * kick" check does not need atomic operations if kvm_vcpu_kick is used
         * within the vCPU thread itself.
         */
        if (vcpu == __this_cpu_read(kvm_running_vcpu)) {
                if (vcpu->mode == IN_GUEST_MODE)
                        WRITE_ONCE(vcpu->mode, EXITING_GUEST_MODE);
                goto out;
        }

        /*
         * Note, the vCPU could get migrated to a different pCPU at any point
         * after kvm_arch_vcpu_should_kick(), which could result in sending an
         * IPI to the previous pCPU.  But, that's ok because the purpose of the
         * IPI is to force the vCPU to leave IN_GUEST_MODE, and migrating the
         * vCPU also requires it to leave IN_GUEST_MODE.
         */
        if (kvm_arch_vcpu_should_kick(vcpu)) {
                cpu = READ_ONCE(vcpu->cpu);
                if (cpu != me && (unsigned int)cpu < nr_cpu_ids && cpu_online(cpu)) {
                        /*
                         * Use a reschedule IPI to kick the vCPU if the caller
                         * doesn't need to wait for a response, as KVM allows
                         * kicking vCPUs while IRQs are disabled, but using the
                         * SMP function call framework with IRQs disabled can
                         * deadlock due to taking cross-CPU locks.
                         */
                        if (wait)
                                smp_call_function_single(cpu, ack_kick, NULL, wait);
                        else
                                smp_send_reschedule(cpu);
                }
        }
out:
        put_cpu();
}
EXPORT_SYMBOL_GPL(__kvm_vcpu_kick);
#endif /* !CONFIG_S390 */

int kvm_vcpu_yield_to(struct kvm_vcpu *target)
{
        struct task_struct *task = NULL;
        int ret;

        if (!read_trylock(&target->pid_lock))
                return 0;

        if (target->pid)
                task = get_pid_task(target->pid, PIDTYPE_PID);

        read_unlock(&target->pid_lock);

        if (!task)
                return 0;
        ret = yield_to(task, 1);
        put_task_struct(task);

        return ret;
}
EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);

/*
 * Helper that checks whether a VCPU is eligible for directed yield.
 * Most eligible candidate to yield is decided by following heuristics:
 *
 *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
 *  (preempted lock holder), indicated by @in_spin_loop.
 *  Set at the beginning and cleared at the end of interception/PLE handler.
 *
 *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
 *  chance last time (mostly it has become eligible now since we have probably
 *  yielded to lockholder in last iteration. This is done by toggling
 *  @dy_eligible each time a VCPU checked for eligibility.)
 *
 *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
 *  to preempted lock-holder could result in wrong VCPU selection and CPU
 *  burning. Giving priority for a potential lock-holder increases lock
 *  progress.
 *
 *  Since algorithm is based on heuristics, accessing another VCPU data without
 *  locking does not harm. It may result in trying to yield to  same VCPU, fail
 *  and continue with next VCPU and so on.
 */
static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
        bool eligible;

        eligible = !vcpu->spin_loop.in_spin_loop ||
                    vcpu->spin_loop.dy_eligible;

        if (vcpu->spin_loop.in_spin_loop)
                kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);

        return eligible;
#else
        return true;
#endif
}

/*
 * Unlike kvm_arch_vcpu_runnable, this function is called outside
 * a vcpu_load/vcpu_put pair.  However, for most architectures
 * kvm_arch_vcpu_runnable does not require vcpu_load.
 */
bool __weak kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
        return kvm_arch_vcpu_runnable(vcpu);
}

static bool vcpu_dy_runnable(struct kvm_vcpu *vcpu)
{
        if (kvm_arch_dy_runnable(vcpu))
                return true;

#ifdef CONFIG_KVM_ASYNC_PF
        if (!list_empty_careful(&vcpu->async_pf.done))
                return true;
#endif

        return false;
}

/*
 * By default, simply query the target vCPU's current mode when checking if a
 * vCPU was preempted in kernel mode.  All architectures except x86 (or more
 * specifical, except VMX) allow querying whether or not a vCPU is in kernel
 * mode even if the vCPU is NOT loaded, i.e. using kvm_arch_vcpu_in_kernel()
 * directly for cross-vCPU checks is functionally correct and accurate.
 */
bool __weak kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu)
{
        return kvm_arch_vcpu_in_kernel(vcpu);
}

bool __weak kvm_arch_dy_has_pending_interrupt(struct kvm_vcpu *vcpu)
{
        return false;
}

void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode)
{
        int nr_vcpus, start, i, idx, yielded;
        struct kvm *kvm = me->kvm;
        struct kvm_vcpu *vcpu;
        int try = 3;

        nr_vcpus = atomic_read(&kvm->online_vcpus);
        if (nr_vcpus < 2)
                return;

        /* Pairs with the smp_wmb() in kvm_vm_ioctl_create_vcpu(). */
        smp_rmb();

        kvm_vcpu_set_in_spin_loop(me, true);

        /*
         * The current vCPU ("me") is spinning in kernel mode, i.e. is likely
         * waiting for a resource to become available.  Attempt to yield to a
         * vCPU that is runnable, but not currently running, e.g. because the
         * vCPU was preempted by a higher priority task.  With luck, the vCPU
         * that was preempted is holding a lock or some other resource that the
         * current vCPU is waiting to acquire, and yielding to the other vCPU
         * will allow it to make forward progress and release the lock (or kick
         * the spinning vCPU, etc).
         *
         * Since KVM has no insight into what exactly the guest is doing,
         * approximate a round-robin selection by iterating over all vCPUs,
         * starting at the last boosted vCPU.  I.e. if N=kvm->last_boosted_vcpu,
         * iterate over vCPU[N+1]..vCPU[N-1], wrapping as needed.
         *
         * Note, this is inherently racy, e.g. if multiple vCPUs are spinning,
         * they may all try to yield to the same vCPU(s).  But as above, this
         * is all best effort due to KVM's lack of visibility into the guest.
         */
        start = READ_ONCE(kvm->last_boosted_vcpu) + 1;
        for (i = 0; i < nr_vcpus; i++) {
                idx = (start + i) % nr_vcpus;
                if (idx == me->vcpu_idx)
                        continue;

                vcpu = xa_load(&kvm->vcpu_array, idx);
                if (!READ_ONCE(vcpu->ready))
                        continue;
                if (kvm_vcpu_is_blocking(vcpu) && !vcpu_dy_runnable(vcpu))
                        continue;

                /*
                 * Treat the target vCPU as being in-kernel if it has a pending
                 * interrupt, as the vCPU trying to yield may be spinning
                 * waiting on IPI delivery, i.e. the target vCPU is in-kernel
                 * for the purposes of directed yield.
                 */
                if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
                    !kvm_arch_dy_has_pending_interrupt(vcpu) &&
                    !kvm_arch_vcpu_preempted_in_kernel(vcpu))
                        continue;

                if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
                        continue;

                yielded = kvm_vcpu_yield_to(vcpu);
                if (yielded > 0) {
                        WRITE_ONCE(kvm->last_boosted_vcpu, i);
                        break;
                } else if (yielded < 0 && !--try) {
                        break;
                }
        }
        kvm_vcpu_set_in_spin_loop(me, false);

        /* Ensure vcpu is not eligible during next spinloop */
        kvm_vcpu_set_dy_eligible(me, false);
}
EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);

static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff)
{
#ifdef CONFIG_HAVE_KVM_DIRTY_RING
        return (pgoff >= KVM_DIRTY_LOG_PAGE_OFFSET) &&
            (pgoff < KVM_DIRTY_LOG_PAGE_OFFSET +
             kvm->dirty_ring_size / PAGE_SIZE);
#else
        return false;
#endif
}

static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf)
{
        struct kvm_vcpu *vcpu = vmf->vma->vm_file->private_data;
        struct page *page;

        if (vmf->pgoff == 0)
                page = virt_to_page(vcpu->run);
#ifdef CONFIG_X86
        else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
                page = virt_to_page(vcpu->arch.pio_data);
#endif
#ifdef CONFIG_KVM_MMIO
        else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
                page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
#endif
        else if (kvm_page_in_dirty_ring(vcpu->kvm, vmf->pgoff))
                page = kvm_dirty_ring_get_page(
                    &vcpu->dirty_ring,
                    vmf->pgoff - KVM_DIRTY_LOG_PAGE_OFFSET);
        else
                return kvm_arch_vcpu_fault(vcpu, vmf);
        get_page(page);
        vmf->page = page;
        return 0;
}

static const struct vm_operations_struct kvm_vcpu_vm_ops = {
        .fault = kvm_vcpu_fault,
};

static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
{
        struct kvm_vcpu *vcpu = file->private_data;
        unsigned long pages = vma_pages(vma);

        if ((kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff) ||
             kvm_page_in_dirty_ring(vcpu->kvm, vma->vm_pgoff + pages - 1)) &&
            ((vma->vm_flags & VM_EXEC) || !(vma->vm_flags & VM_SHARED)))
                return -EINVAL;

        vma->vm_ops = &kvm_vcpu_vm_ops;
        return 0;
}

static int kvm_vcpu_release(struct inode *inode, struct file *filp)
{
        struct kvm_vcpu *vcpu = filp->private_data;

        kvm_put_kvm(vcpu->kvm);
        return 0;
}

static struct file_operations kvm_vcpu_fops = {
        .release        = kvm_vcpu_release,
        .unlocked_ioctl = kvm_vcpu_ioctl,
        .mmap           = kvm_vcpu_mmap,
        .llseek                = noop_llseek,
        KVM_COMPAT(kvm_vcpu_compat_ioctl),
};

/*
 * Allocates an inode for the vcpu.
 */
static int create_vcpu_fd(struct kvm_vcpu *vcpu)
{
        char name[8 + 1 + ITOA_MAX_LEN + 1];

        snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id);
        return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
}

#ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS
static int vcpu_get_pid(void *data, u64 *val)
{
        struct kvm_vcpu *vcpu = data;

        read_lock(&vcpu->pid_lock);
        *val = pid_nr(vcpu->pid);
        read_unlock(&vcpu->pid_lock);
        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vcpu_get_pid_fops, vcpu_get_pid, NULL, "%llu\n");

static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
{
        struct dentry *debugfs_dentry;
        char dir_name[ITOA_MAX_LEN * 2];

        if (!debugfs_initialized())
                return;

        snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
        debugfs_dentry = debugfs_create_dir(dir_name,
                                            vcpu->kvm->debugfs_dentry);
        debugfs_create_file("pid", 0444, debugfs_dentry, vcpu,
                            &vcpu_get_pid_fops);

        kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry);
}
#endif

/*
 * Creates some virtual cpus.  Good luck creating more than one.
 */
static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id)
{
        int r;
        struct kvm_vcpu *vcpu;
        struct page *page;

        /*
         * KVM tracks vCPU IDs as 'int', be kind to userspace and reject
         * too-large values instead of silently truncating.
         *
         * Ensure KVM_MAX_VCPU_IDS isn't pushed above INT_MAX without first
         * changing the storage type (at the very least, IDs should be tracked
         * as unsigned ints).
         */
        BUILD_BUG_ON(KVM_MAX_VCPU_IDS > INT_MAX);
        if (id >= KVM_MAX_VCPU_IDS)
                return -EINVAL;

        mutex_lock(&kvm->lock);
        if (kvm->created_vcpus >= kvm->max_vcpus) {
                mutex_unlock(&kvm->lock);
                return -EINVAL;
        }

        r = kvm_arch_vcpu_precreate(kvm, id);
        if (r) {
                mutex_unlock(&kvm->lock);
                return r;
        }

        kvm->created_vcpus++;
        mutex_unlock(&kvm->lock);

        vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT);
        if (!vcpu) {
                r = -ENOMEM;
                goto vcpu_decrement;
        }

        BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE);
        page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
        if (!page) {
                r = -ENOMEM;
                goto vcpu_free;
        }
        vcpu->run = page_address(page);

        kvm_vcpu_init(vcpu, kvm, id);

        r = kvm_arch_vcpu_create(vcpu);
        if (r)
                goto vcpu_free_run_page;

        if (kvm->dirty_ring_size) {
                r = kvm_dirty_ring_alloc(kvm, &vcpu->dirty_ring,
                                         id, kvm->dirty_ring_size);
                if (r)
                        goto arch_vcpu_destroy;
        }

        mutex_lock(&kvm->lock);

        if (kvm_get_vcpu_by_id(kvm, id)) {
                r = -EEXIST;
                goto unlock_vcpu_destroy;
        }

        vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus);
        r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT);
        WARN_ON_ONCE(r == -EBUSY);
        if (r)
                goto unlock_vcpu_destroy;

        /*
         * Now it's all set up, let userspace reach it.  Grab the vCPU's mutex
         * so that userspace can't invoke vCPU ioctl()s until the vCPU is fully
         * visible (per online_vcpus), e.g. so that KVM doesn't get tricked
         * into a NULL-pointer dereference because KVM thinks the _current_
         * vCPU doesn't exist.  As a bonus, taking vcpu->mutex ensures lockdep
         * knows it's taken *inside* kvm->lock.
         */
        mutex_lock(&vcpu->mutex);
        kvm_get_kvm(kvm);
        r = create_vcpu_fd(vcpu);
        if (r < 0)
                goto kvm_put_xa_erase;

        /*
         * Pairs with smp_rmb() in kvm_get_vcpu.  Store the vcpu
         * pointer before kvm->online_vcpu's incremented value.
         */
        smp_wmb();
        atomic_inc(&kvm->online_vcpus);
        mutex_unlock(&vcpu->mutex);

        mutex_unlock(&kvm->lock);
        kvm_arch_vcpu_postcreate(vcpu);
        kvm_create_vcpu_debugfs(vcpu);
        return r;

kvm_put_xa_erase:
        mutex_unlock(&vcpu->mutex);
        kvm_put_kvm_no_destroy(kvm);
        xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx);
unlock_vcpu_destroy:
        mutex_unlock(&kvm->lock);
        kvm_dirty_ring_free(&vcpu->dirty_ring);
arch_vcpu_destroy:
        kvm_arch_vcpu_destroy(vcpu);
vcpu_free_run_page:
        free_page((unsigned long)vcpu->run);
vcpu_free:
        kmem_cache_free(kvm_vcpu_cache, vcpu);
vcpu_decrement:
        mutex_lock(&kvm->lock);
        kvm->created_vcpus--;
        mutex_unlock(&kvm->lock);
        return r;
}

static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
{
        if (sigset) {
                sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
                vcpu->sigset_active = 1;
                vcpu->sigset = *sigset;
        } else
                vcpu->sigset_active = 0;
        return 0;
}

static ssize_t kvm_vcpu_stats_read(struct file *file, char __user *user_buffer,
                              size_t size, loff_t *offset)
{
        struct kvm_vcpu *vcpu = file->private_data;

        return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header,
                        &kvm_vcpu_stats_desc[0], &vcpu->stat,
                        sizeof(vcpu->stat), user_buffer, size, offset);
}

static int kvm_vcpu_stats_release(struct inode *inode, struct file *file)
{
        struct kvm_vcpu *vcpu = file->private_data;

        kvm_put_kvm(vcpu->kvm);
        return 0;
}

static const struct file_operations kvm_vcpu_stats_fops = {
        .owner = THIS_MODULE,
        .read = kvm_vcpu_stats_read,
        .release = kvm_vcpu_stats_release,
        .llseek = noop_llseek,
};

static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu)
{
        int fd;
        struct file *file;
        char name[15 + ITOA_MAX_LEN + 1];

        snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id);

        fd = get_unused_fd_flags(O_CLOEXEC);
        if (fd < 0)
                return fd;

        file = anon_inode_getfile_fmode(name, &kvm_vcpu_stats_fops, vcpu,
                                        O_RDONLY, FMODE_PREAD);
        if (IS_ERR(file)) {
                put_unused_fd(fd);
                return PTR_ERR(file);
        }

        kvm_get_kvm(vcpu->kvm);
        fd_install(fd, file);

        return fd;
}

#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
                                     struct kvm_pre_fault_memory *range)
{
        int idx;
        long r;
        u64 full_size;

        if (range->flags)
                return -EINVAL;

        if (!PAGE_ALIGNED(range->gpa) ||
            !PAGE_ALIGNED(range->size) ||
            range->gpa + range->size <= range->gpa)
                return -EINVAL;

        vcpu_load(vcpu);
        idx = srcu_read_lock(&vcpu->kvm->srcu);

        full_size = range->size;
        do {
                if (signal_pending(current)) {
                        r = -EINTR;
                        break;
                }

                r = kvm_arch_vcpu_pre_fault_memory(vcpu, range);
                if (WARN_ON_ONCE(r == 0 || r == -EIO))
                        break;

                if (r < 0)
                        break;

                range->size -= r;
                range->gpa += r;
                cond_resched();
        } while (range->size);

        srcu_read_unlock(&vcpu->kvm->srcu, idx);
        vcpu_put(vcpu);

        /* Return success if at least one page was mapped successfully.  */
        return full_size == range->size ? r : 0;
}
#endif

static int kvm_wait_for_vcpu_online(struct kvm_vcpu *vcpu)
{
        struct kvm *kvm = vcpu->kvm;

        /*
         * In practice, this happy path will always be taken, as a well-behaved
         * VMM will never invoke a vCPU ioctl() before KVM_CREATE_VCPU returns.
         */
        if (likely(vcpu->vcpu_idx < atomic_read(&kvm->online_vcpus)))
                return 0;

        /*
         * Acquire and release the vCPU's mutex to wait for vCPU creation to
         * complete (kvm_vm_ioctl_create_vcpu() holds the mutex until the vCPU
         * is fully online).
         */
        if (mutex_lock_killable(&vcpu->mutex))
                return -EINTR;

        mutex_unlock(&vcpu->mutex);

        if (WARN_ON_ONCE(!kvm_get_vcpu(kvm, vcpu->vcpu_idx)))
                return -EIO;

        return 0;
}

static long kvm_vcpu_ioctl(struct file *filp,
                           unsigned int ioctl, unsigned long arg)
{
        struct kvm_vcpu *vcpu = filp->private_data;
        void __user *argp = (void __user *)arg;
        int r;
        struct kvm_fpu *fpu = NULL;
        struct kvm_sregs *kvm_sregs = NULL;

        if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
                return -EIO;

        if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
                return -EINVAL;

        /*
         * Wait for the vCPU to be online before handling the ioctl(), as KVM
         * assumes the vCPU is reachable via vcpu_array, i.e. may dereference
         * a NULL pointer if userspace invokes an ioctl() before KVM is ready.
         */
        r = kvm_wait_for_vcpu_online(vcpu);
        if (r)
                return r;

        /*
         * Some architectures have vcpu ioctls that are asynchronous to vcpu
         * execution; mutex_lock() would break them.
         */
        r = kvm_arch_vcpu_async_ioctl(filp, ioctl, arg);
        if (r != -ENOIOCTLCMD)
                return r;

        if (mutex_lock_killable(&vcpu->mutex))
                return -EINTR;
        switch (ioctl) {
        case KVM_RUN: {
                struct pid *oldpid;
                r = -EINVAL;
                if (arg)
                        goto out;

                /*
                 * Note, vcpu->pid is primarily protected by vcpu->mutex. The
                 * dedicated r/w lock allows other tasks, e.g. other vCPUs, to
                 * read vcpu->pid while this vCPU is in KVM_RUN, e.g. to yield
                 * directly to this vCPU
                 */
                oldpid = vcpu->pid;
                if (unlikely(oldpid != task_pid(current))) {
                        /* The thread running this VCPU changed. */
                        struct pid *newpid;

                        r = kvm_arch_vcpu_run_pid_change(vcpu);
                        if (r)
                                break;

                        newpid = get_task_pid(current, PIDTYPE_PID);
                        write_lock(&vcpu->pid_lock);
                        vcpu->pid = newpid;
                        write_unlock(&vcpu->pid_lock);

                        put_pid(oldpid);
                }
                vcpu->wants_to_run = !READ_ONCE(vcpu->run->immediate_exit__unsafe);
                r = kvm_arch_vcpu_ioctl_run(vcpu);
                vcpu->wants_to_run = false;

                trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
                break;
        }
        case KVM_GET_REGS: {
                struct kvm_regs *kvm_regs;

                r = -ENOMEM;
                kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
                if (!kvm_regs)
                        goto out;
                r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
                if (r)
                        goto out_free1;
                r = -EFAULT;
                if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
                        goto out_free1;
                r = 0;
out_free1:
                kfree(kvm_regs);
                break;
        }
        case KVM_SET_REGS: {
                struct kvm_regs *kvm_regs;

                kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
                if (IS_ERR(kvm_regs)) {
                        r = PTR_ERR(kvm_regs);
                        goto out;
                }
                r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
                kfree(kvm_regs);
                break;
        }
        case KVM_GET_SREGS: {
                kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
                r = -ENOMEM;
                if (!kvm_sregs)
                        goto out;
                r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
                if (r)
                        goto out;
                r = -EFAULT;
                if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
                        goto out;
                r = 0;
                break;
        }
        case KVM_SET_SREGS: {
                kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
                if (IS_ERR(kvm_sregs)) {
                        r = PTR_ERR(kvm_sregs);
                        kvm_sregs = NULL;
                        goto out;
                }
                r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
                break;
        }
        case KVM_GET_MP_STATE: {
                struct kvm_mp_state mp_state;

                r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
                if (r)
                        goto out;
                r = -EFAULT;
                if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
                        goto out;
                r = 0;
                break;
        }
        case KVM_SET_MP_STATE: {
                struct kvm_mp_state mp_state;

                r = -EFAULT;
                if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
                        goto out;
                r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
                break;
        }
        case KVM_TRANSLATE: {
                struct kvm_translation tr;

                r = -EFAULT;
                if (copy_from_user(&tr, argp, sizeof(tr)))
                        goto out;
                r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
                if (r)
                        goto out;
                r = -EFAULT;
                if (copy_to_user(argp, &tr, sizeof(tr)))
                        goto out;
                r = 0;
                break;
        }
        case KVM_SET_GUEST_DEBUG: {
                struct kvm_guest_debug dbg;

                r = -EFAULT;
                if (copy_from_user(&dbg, argp, sizeof(dbg)))
                        goto out;
                r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
                break;
        }
        case KVM_SET_SIGNAL_MASK: {
                struct kvm_signal_mask __user *sigmask_arg = argp;
                struct kvm_signal_mask kvm_sigmask;
                sigset_t sigset, *p;

                p = NULL;
                if (argp) {
                        r = -EFAULT;
                        if (copy_from_user(&kvm_sigmask, argp,
                                           sizeof(kvm_sigmask)))
                                goto out;
                        r = -EINVAL;
                        if (kvm_sigmask.len != sizeof(sigset))
                                goto out;
                        r = -EFAULT;
                        if (copy_from_user(&sigset, sigmask_arg->sigset,
                                           sizeof(sigset)))
                                goto out;
                        p = &sigset;
                }
                r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
                break;
        }
        case KVM_GET_FPU: {
                fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
                r = -ENOMEM;
                if (!fpu)
                        goto out;
                r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
                if (r)
                        goto out;
                r = -EFAULT;
                if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
                        goto out;
                r = 0;
                break;
        }
        case KVM_SET_FPU: {
                fpu = memdup_user(argp, sizeof(*fpu));
                if (IS_ERR(fpu)) {
                        r = PTR_ERR(fpu);
                        fpu = NULL;
                        goto out;
                }
                r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
                break;
        }
        case KVM_GET_STATS_FD: {
                r = kvm_vcpu_ioctl_get_stats_fd(vcpu);
                break;
        }
#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY
        case KVM_PRE_FAULT_MEMORY: {
                struct kvm_pre_fault_memory range;

                r = -EFAULT;
                if (copy_from_user(&range, argp, sizeof(range)))
                        break;
                r = kvm_vcpu_pre_fault_memory(vcpu, &range);
                /* Pass back leftover range. */
                if (copy_to_user(argp, &range, sizeof(range)))
                        r = -EFAULT;
                break;
        }
#endif
        default:
                r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
        }
out:
        mutex_unlock(&vcpu->mutex);
        kfree(fpu);
        kfree(kvm_sregs);
        return r;
}

#ifdef CONFIG_KVM_COMPAT
static long kvm_vcpu_compat_ioctl(struct file *filp,
                                  unsigned int ioctl, unsigned long arg)
{
        struct kvm_vcpu *vcpu = filp->private_data;
        void __user *argp = compat_ptr(arg);
        int r;

        if (vcpu->kvm->mm != current->mm || vcpu->kvm->vm_dead)
                return -EIO;

        switch (ioctl) {
        case KVM_SET_SIGNAL_MASK: {
                struct kvm_signal_mask __user *sigmask_arg = argp;
                struct kvm_signal_mask kvm_sigmask;
                sigset_t sigset;

                if (argp) {
                        r = -EFAULT;
                        if (copy_from_user(&kvm_sigmask, argp,
                                           sizeof(kvm_sigmask)))
                                goto out;
                        r = -EINVAL;
                        if (kvm_sigmask.len != sizeof(compat_sigset_t))
                                goto out;
                        r = -EFAULT;
                        if (get_compat_sigset(&sigset,
                                              (compat_sigset_t __user *)sigmask_arg->sigset))
                                goto out;
                        r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
                } else
                        r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
                break;
        }
        default:
                r = kvm_vcpu_ioctl(filp, ioctl, arg);
        }

out:
        return r;
}
#endif

static int kvm_device_mmap(struct file *filp, struct vm_area_struct *vma)
{
        struct kvm_device *dev = filp->private_data;

        if (dev->ops->mmap)
                return dev->ops->mmap(dev, vma);

        return -ENODEV;
}

static int kvm_device_ioctl_attr(struct kvm_device *dev,
                                 int (*accessor)(struct kvm_device *dev,
                                                 struct kvm_device_attr *attr),
                                 unsigned long arg)
{
        struct kvm_device_attr attr;

        if (!accessor)
                return -EPERM;

        if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
                return -EFAULT;

        return accessor(dev, &attr);
}

static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
                             unsigned long arg)
{
        struct kvm_device *dev = filp->private_data;

        if (dev->kvm->mm != current->mm || dev->kvm->vm_dead)
                return -EIO;

        switch (ioctl) {
        case KVM_SET_DEVICE_ATTR:
                return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
        case KVM_GET_DEVICE_ATTR:
                return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
        case KVM_HAS_DEVICE_ATTR:
                return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
        default:
                if (dev->ops->ioctl)
                        return dev->ops->ioctl(dev, ioctl, arg);

                return -ENOTTY;
        }
}

static int kvm_device_release(struct inode *inode, struct file *filp)
{
        struct kvm_device *dev = filp->private_data;
        struct kvm *kvm = dev->kvm;

        if (dev->ops->release) {
                mutex_lock(&kvm->lock);
                list_del_rcu(&dev->vm_node);
                synchronize_rcu();
                dev->ops->release(dev);
                mutex_unlock(&kvm->lock);
        }

        kvm_put_kvm(kvm);
        return 0;
}

static struct file_operations kvm_device_fops = {
        .unlocked_ioctl = kvm_device_ioctl,
        .release = kvm_device_release,
        KVM_COMPAT(kvm_device_ioctl),
        .mmap = kvm_device_mmap,
};

struct kvm_device *kvm_device_from_filp(struct file *filp)
{
        if (filp->f_op != &kvm_device_fops)
                return NULL;

        return filp->private_data;
}

static const struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
#ifdef CONFIG_KVM_MPIC
        [KVM_DEV_TYPE_FSL_MPIC_20]        = &kvm_mpic_ops,
        [KVM_DEV_TYPE_FSL_MPIC_42]        = &kvm_mpic_ops,
#endif
};

int kvm_register_device_ops(const struct kvm_device_ops *ops, u32 type)
{
        if (type >= ARRAY_SIZE(kvm_device_ops_table))
                return -ENOSPC;

        if (kvm_device_ops_table[type] != NULL)
                return -EEXIST;

        kvm_device_ops_table[type] = ops;
        return 0;
}

void kvm_unregister_device_ops(u32 type)
{
        if (kvm_device_ops_table[type] != NULL)
                kvm_device_ops_table[type] = NULL;
}

static int kvm_ioctl_create_device(struct kvm *kvm,
                                   struct kvm_create_device *cd)
{
        const struct kvm_device_ops *ops;
        struct kvm_device *dev;
        bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
        int type;
        int ret;

        if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
                return -ENODEV;

        type = array_index_nospec(cd->type, ARRAY_SIZE(kvm_device_ops_table));
        ops = kvm_device_ops_table[type];
        if (ops == NULL)
                return -ENODEV;

        if (test)
                return 0;

        dev = kzalloc(sizeof(*dev), GFP_KERNEL_ACCOUNT);
        if (!dev)
                return -ENOMEM;

        dev->ops = ops;
        dev->kvm = kvm;

        mutex_lock(&kvm->lock);
        ret = ops->create(dev, type);
        if (ret < 0) {
                mutex_unlock(&kvm->lock);
                kfree(dev);
                return ret;
        }
        list_add_rcu(&dev->vm_node, &kvm->devices);
        mutex_unlock(&kvm->lock);

        if (ops->init)
                ops->init(dev);

        kvm_get_kvm(kvm);
        ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
        if (ret < 0) {
                kvm_put_kvm_no_destroy(kvm);
                mutex_lock(&kvm->lock);
                list_del_rcu(&dev->vm_node);
                synchronize_rcu();
                if (ops->release)
                        ops->release(dev);
                mutex_unlock(&kvm->lock);
                if (ops->destroy)
                        ops->destroy(dev);
                return ret;
        }

        cd->fd = ret;
        return 0;
}

static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
{
        switch (arg) {
        case KVM_CAP_USER_MEMORY:
        case KVM_CAP_USER_MEMORY2:
        case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
        case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
        case KVM_CAP_INTERNAL_ERROR_DATA:
#ifdef CONFIG_HAVE_KVM_MSI
        case KVM_CAP_SIGNAL_MSI:
#endif
#ifdef CONFIG_HAVE_KVM_IRQCHIP
        case KVM_CAP_IRQFD:
#endif
        case KVM_CAP_IOEVENTFD_ANY_LENGTH:
        case KVM_CAP_CHECK_EXTENSION_VM:
        case KVM_CAP_ENABLE_CAP_VM:
        case KVM_CAP_HALT_POLL:
                return 1;
#ifdef CONFIG_KVM_MMIO
        case KVM_CAP_COALESCED_MMIO:
                return KVM_COALESCED_MMIO_PAGE_OFFSET;
        case KVM_CAP_COALESCED_PIO:
                return 1;
#endif
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
        case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2:
                return KVM_DIRTY_LOG_MANUAL_CAPS;
#endif
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
        case KVM_CAP_IRQ_ROUTING:
                return KVM_MAX_IRQ_ROUTES;
#endif
#if KVM_MAX_NR_ADDRESS_SPACES > 1
        case KVM_CAP_MULTI_ADDRESS_SPACE:
                if (kvm)
                        return kvm_arch_nr_memslot_as_ids(kvm);
                return KVM_MAX_NR_ADDRESS_SPACES;
#endif
        case KVM_CAP_NR_MEMSLOTS:
                return KVM_USER_MEM_SLOTS;
        case KVM_CAP_DIRTY_LOG_RING:
#ifdef CONFIG_HAVE_KVM_DIRTY_RING_TSO
                return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
#else
                return 0;
#endif
        case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
#ifdef CONFIG_HAVE_KVM_DIRTY_RING_ACQ_REL
                return KVM_DIRTY_RING_MAX_ENTRIES * sizeof(struct kvm_dirty_gfn);
#else
                return 0;
#endif
#ifdef CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP
        case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP:
#endif
        case KVM_CAP_BINARY_STATS_FD:
        case KVM_CAP_SYSTEM_EVENT_DATA:
        case KVM_CAP_DEVICE_CTRL:
                return 1;
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
        case KVM_CAP_MEMORY_ATTRIBUTES:
                return kvm_supported_mem_attributes(kvm);
#endif
#ifdef CONFIG_KVM_PRIVATE_MEM
        case KVM_CAP_GUEST_MEMFD:
                return !kvm || kvm_arch_has_private_mem(kvm);
#endif
        default:
                break;
        }
        return kvm_vm_ioctl_check_extension(kvm, arg);
}

static int kvm_vm_ioctl_enable_dirty_log_ring(struct kvm *kvm, u32 size)
{
        int r;

        if (!KVM_DIRTY_LOG_PAGE_OFFSET)
                return -EINVAL;

        /* the size should be power of 2 */
        if (!size || (size & (size - 1)))
                return -EINVAL;

        /* Should be bigger to keep the reserved entries, or a page */
        if (size < kvm_dirty_ring_get_rsvd_entries(kvm) *
            sizeof(struct kvm_dirty_gfn) || size < PAGE_SIZE)
                return -EINVAL;

        if (size > KVM_DIRTY_RING_MAX_ENTRIES *
            sizeof(struct kvm_dirty_gfn))
                return -E2BIG;

        /* We only allow it to set once */
        if (kvm->dirty_ring_size)
                return -EINVAL;

        mutex_lock(&kvm->lock);

        if (kvm->created_vcpus) {
                /* We don't allow to change this value after vcpu created */
                r = -EINVAL;
        } else {
                kvm->dirty_ring_size = size;
                r = 0;
        }

        mutex_unlock(&kvm->lock);
        return r;
}

static int kvm_vm_ioctl_reset_dirty_pages(struct kvm *kvm)
{
        unsigned long i;
        struct kvm_vcpu *vcpu;
        int cleared = 0;

        if (!kvm->dirty_ring_size)
                return -EINVAL;

        mutex_lock(&kvm->slots_lock);

        kvm_for_each_vcpu(i, vcpu, kvm)
                cleared += kvm_dirty_ring_reset(vcpu->kvm, &vcpu->dirty_ring);

        mutex_unlock(&kvm->slots_lock);

        if (cleared)
                kvm_flush_remote_tlbs(kvm);

        return cleared;
}

int __attribute__((weak)) kvm_vm_ioctl_enable_cap(struct kvm *kvm,
                                                  struct kvm_enable_cap *cap)
{
        return -EINVAL;
}

bool kvm_are_all_memslots_empty(struct kvm *kvm)
{
        int i;

        lockdep_assert_held(&kvm->slots_lock);

        for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) {
                if (!kvm_memslots_empty(__kvm_memslots(kvm, i)))
                        return false;
        }

        return true;
}
EXPORT_SYMBOL_GPL(kvm_are_all_memslots_empty);

static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm,
                                           struct kvm_enable_cap *cap)
{
        switch (cap->cap) {
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
        case KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2: {
                u64 allowed_options = KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE;

                if (cap->args[0] & KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE)
                        allowed_options = KVM_DIRTY_LOG_MANUAL_CAPS;

                if (cap->flags || (cap->args[0] & ~allowed_options))
                        return -EINVAL;
                kvm->manual_dirty_log_protect = cap->args[0];
                return 0;
        }
#endif
        case KVM_CAP_HALT_POLL: {
                if (cap->flags || cap->args[0] != (unsigned int)cap->args[0])
                        return -EINVAL;

                kvm->max_halt_poll_ns = cap->args[0];

                /*
                 * Ensure kvm->override_halt_poll_ns does not become visible
                 * before kvm->max_halt_poll_ns.
                 *
                 * Pairs with the smp_rmb() in kvm_vcpu_max_halt_poll_ns().
                 */
                smp_wmb();
                kvm->override_halt_poll_ns = true;

                return 0;
        }
        case KVM_CAP_DIRTY_LOG_RING:
        case KVM_CAP_DIRTY_LOG_RING_ACQ_REL:
                if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap))
                        return -EINVAL;

                return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]);
        case KVM_CAP_DIRTY_LOG_RING_WITH_BITMAP: {
                int r = -EINVAL;

                if (!IS_ENABLED(CONFIG_NEED_KVM_DIRTY_RING_WITH_BITMAP) ||
                    !kvm->dirty_ring_size || cap->flags)
                        return r;

                mutex_lock(&kvm->slots_lock);

                /*
                 * For simplicity, allow enabling ring+bitmap if and only if
                 * there are no memslots, e.g. to ensure all memslots allocate
                 * a bitmap after the capability is enabled.
                 */
                if (kvm_are_all_memslots_empty(kvm)) {
                        kvm->dirty_ring_with_bitmap = true;
                        r = 0;
                }

                mutex_unlock(&kvm->slots_lock);

                return r;
        }
        default:
                return kvm_vm_ioctl_enable_cap(kvm, cap);
        }
}

static ssize_t kvm_vm_stats_read(struct file *file, char __user *user_buffer,
                              size_t size, loff_t *offset)
{
        struct kvm *kvm = file->private_data;

        return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header,
                                &kvm_vm_stats_desc[0], &kvm->stat,
                                sizeof(kvm->stat), user_buffer, size, offset);
}

static int kvm_vm_stats_release(struct inode *inode, struct file *file)
{
        struct kvm *kvm = file->private_data;

        kvm_put_kvm(kvm);
        return 0;
}

static const struct file_operations kvm_vm_stats_fops = {
        .owner = THIS_MODULE,
        .read = kvm_vm_stats_read,
        .release = kvm_vm_stats_release,
        .llseek = noop_llseek,
};

static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm)
{
        int fd;
        struct file *file;

        fd = get_unused_fd_flags(O_CLOEXEC);
        if (fd < 0)
                return fd;

        file = anon_inode_getfile_fmode("kvm-vm-stats",
                        &kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD);
        if (IS_ERR(file)) {
                put_unused_fd(fd);
                return PTR_ERR(file);
        }

        kvm_get_kvm(kvm);
        fd_install(fd, file);

        return fd;
}

#define SANITY_CHECK_MEM_REGION_FIELD(field)                                        \
do {                                                                                \
        BUILD_BUG_ON(offsetof(struct kvm_userspace_memory_region, field) !=                \
                     offsetof(struct kvm_userspace_memory_region2, field));        \
        BUILD_BUG_ON(sizeof_field(struct kvm_userspace_memory_region, field) !=                \
                     sizeof_field(struct kvm_userspace_memory_region2, field));        \
} while (0)

static long kvm_vm_ioctl(struct file *filp,
                           unsigned int ioctl, unsigned long arg)
{
        struct kvm *kvm = filp->private_data;
        void __user *argp = (void __user *)arg;
        int r;

        if (kvm->mm != current->mm || kvm->vm_dead)
                return -EIO;
        switch (ioctl) {
        case KVM_CREATE_VCPU:
                r = kvm_vm_ioctl_create_vcpu(kvm, arg);
                break;
        case KVM_ENABLE_CAP: {
                struct kvm_enable_cap cap;

                r = -EFAULT;
                if (copy_from_user(&cap, argp, sizeof(cap)))
                        goto out;
                r = kvm_vm_ioctl_enable_cap_generic(kvm, &cap);
                break;
        }
        case KVM_SET_USER_MEMORY_REGION2:
        case KVM_SET_USER_MEMORY_REGION: {
                struct kvm_userspace_memory_region2 mem;
                unsigned long size;

                if (ioctl == KVM_SET_USER_MEMORY_REGION) {
                        /*
                         * Fields beyond struct kvm_userspace_memory_region shouldn't be
                         * accessed, but avoid leaking kernel memory in case of a bug.
                         */
                        memset(&mem, 0, sizeof(mem));
                        size = sizeof(struct kvm_userspace_memory_region);
                } else {
                        size = sizeof(struct kvm_userspace_memory_region2);
                }

                /* Ensure the common parts of the two structs are identical. */
                SANITY_CHECK_MEM_REGION_FIELD(slot);
                SANITY_CHECK_MEM_REGION_FIELD(flags);
                SANITY_CHECK_MEM_REGION_FIELD(guest_phys_addr);
                SANITY_CHECK_MEM_REGION_FIELD(memory_size);
                SANITY_CHECK_MEM_REGION_FIELD(userspace_addr);

                r = -EFAULT;
                if (copy_from_user(&mem, argp, size))
                        goto out;

                r = -EINVAL;
                if (ioctl == KVM_SET_USER_MEMORY_REGION &&
                    (mem.flags & ~KVM_SET_USER_MEMORY_REGION_V1_FLAGS))
                        goto out;

                r = kvm_vm_ioctl_set_memory_region(kvm, &mem);
                break;
        }
        case KVM_GET_DIRTY_LOG: {
                struct kvm_dirty_log log;

                r = -EFAULT;
                if (copy_from_user(&log, argp, sizeof(log)))
                        goto out;
                r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
                break;
        }
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
        case KVM_CLEAR_DIRTY_LOG: {
                struct kvm_clear_dirty_log log;

                r = -EFAULT;
                if (copy_from_user(&log, argp, sizeof(log)))
                        goto out;
                r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
                break;
        }
#endif
#ifdef CONFIG_KVM_MMIO
        case KVM_REGISTER_COALESCED_MMIO: {
                struct kvm_coalesced_mmio_zone zone;

                r = -EFAULT;
                if (copy_from_user(&zone, argp, sizeof(zone)))
                        goto out;
                r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
                break;
        }
        case KVM_UNREGISTER_COALESCED_MMIO: {
                struct kvm_coalesced_mmio_zone zone;

                r = -EFAULT;
                if (copy_from_user(&zone, argp, sizeof(zone)))
                        goto out;
                r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
                break;
        }
#endif
        case KVM_IRQFD: {
                struct kvm_irqfd data;

                r = -EFAULT;
                if (copy_from_user(&data, argp, sizeof(data)))
                        goto out;
                r = kvm_irqfd(kvm, &data);
                break;
        }
        case KVM_IOEVENTFD: {
                struct kvm_ioeventfd data;

                r = -EFAULT;
                if (copy_from_user(&data, argp, sizeof(data)))
                        goto out;
                r = kvm_ioeventfd(kvm, &data);
                break;
        }
#ifdef CONFIG_HAVE_KVM_MSI
        case KVM_SIGNAL_MSI: {
                struct kvm_msi msi;

                r = -EFAULT;
                if (copy_from_user(&msi, argp, sizeof(msi)))
                        goto out;
                r = kvm_send_userspace_msi(kvm, &msi);
                break;
        }
#endif
#ifdef __KVM_HAVE_IRQ_LINE
        case KVM_IRQ_LINE_STATUS:
        case KVM_IRQ_LINE: {
                struct kvm_irq_level irq_event;

                r = -EFAULT;
                if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
                        goto out;

                r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
                                        ioctl == KVM_IRQ_LINE_STATUS);
                if (r)
                        goto out;

                r = -EFAULT;
                if (ioctl == KVM_IRQ_LINE_STATUS) {
                        if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
                                goto out;
                }

                r = 0;
                break;
        }
#endif
#ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
        case KVM_SET_GSI_ROUTING: {
                struct kvm_irq_routing routing;
                struct kvm_irq_routing __user *urouting;
                struct kvm_irq_routing_entry *entries = NULL;

                r = -EFAULT;
                if (copy_from_user(&routing, argp, sizeof(routing)))
                        goto out;
                r = -EINVAL;
                if (!kvm_arch_can_set_irq_routing(kvm))
                        goto out;
                if (routing.nr > KVM_MAX_IRQ_ROUTES)
                        goto out;
                if (routing.flags)
                        goto out;
                if (routing.nr) {
                        urouting = argp;
                        entries = vmemdup_array_user(urouting->entries,
                                                     routing.nr, sizeof(*entries));
                        if (IS_ERR(entries)) {
                                r = PTR_ERR(entries);
                                goto out;
                        }
                }
                r = kvm_set_irq_routing(kvm, entries, routing.nr,
                                        routing.flags);
                kvfree(entries);
                break;
        }
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
        case KVM_SET_MEMORY_ATTRIBUTES: {
                struct kvm_memory_attributes attrs;

                r = -EFAULT;
                if (copy_from_user(&attrs, argp, sizeof(attrs)))
                        goto out;

                r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
                break;
        }
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
        case KVM_CREATE_DEVICE: {
                struct kvm_create_device cd;

                r = -EFAULT;
                if (copy_from_user(&cd, argp, sizeof(cd)))
                        goto out;

                r = kvm_ioctl_create_device(kvm, &cd);
                if (r)
                        goto out;

                r = -EFAULT;
                if (copy_to_user(argp, &cd, sizeof(cd)))
                        goto out;

                r = 0;
                break;
        }
        case KVM_CHECK_EXTENSION:
                r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
                break;
        case KVM_RESET_DIRTY_RINGS:
                r = kvm_vm_ioctl_reset_dirty_pages(kvm);
                break;
        case KVM_GET_STATS_FD:
                r = kvm_vm_ioctl_get_stats_fd(kvm);
                break;
#ifdef CONFIG_KVM_PRIVATE_MEM
        case KVM_CREATE_GUEST_MEMFD: {
                struct kvm_create_guest_memfd guest_memfd;

                r = -EFAULT;
                if (copy_from_user(&guest_memfd, argp, sizeof(guest_memfd)))
                        goto out;

                r = kvm_gmem_create(kvm, &guest_memfd);
                break;
        }
#endif
        default:
                r = kvm_arch_vm_ioctl(filp, ioctl, arg);
        }
out:
        return r;
}

#ifdef CONFIG_KVM_COMPAT
struct compat_kvm_dirty_log {
        __u32 slot;
        __u32 padding1;
        union {
                compat_uptr_t dirty_bitmap; /* one bit per page */
                __u64 padding2;
        };
};

struct compat_kvm_clear_dirty_log {
        __u32 slot;
        __u32 num_pages;
        __u64 first_page;
        union {
                compat_uptr_t dirty_bitmap; /* one bit per page */
                __u64 padding2;
        };
};

long __weak kvm_arch_vm_compat_ioctl(struct file *filp, unsigned int ioctl,
                                     unsigned long arg)
{
        return -ENOTTY;
}

static long kvm_vm_compat_ioctl(struct file *filp,
                           unsigned int ioctl, unsigned long arg)
{
        struct kvm *kvm = filp->private_data;
        int r;

        if (kvm->mm != current->mm || kvm->vm_dead)
                return -EIO;

        r = kvm_arch_vm_compat_ioctl(filp, ioctl, arg);
        if (r != -ENOTTY)
                return r;

        switch (ioctl) {
#ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
        case KVM_CLEAR_DIRTY_LOG: {
                struct compat_kvm_clear_dirty_log compat_log;
                struct kvm_clear_dirty_log log;

                if (copy_from_user(&compat_log, (void __user *)arg,
                                   sizeof(compat_log)))
                        return -EFAULT;
                log.slot         = compat_log.slot;
                log.num_pages         = compat_log.num_pages;
                log.first_page         = compat_log.first_page;
                log.padding2         = compat_log.padding2;
                log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

                r = kvm_vm_ioctl_clear_dirty_log(kvm, &log);
                break;
        }
#endif
        case KVM_GET_DIRTY_LOG: {
                struct compat_kvm_dirty_log compat_log;
                struct kvm_dirty_log log;

                if (copy_from_user(&compat_log, (void __user *)arg,
                                   sizeof(compat_log)))
                        return -EFAULT;
                log.slot         = compat_log.slot;
                log.padding1         = compat_log.padding1;
                log.padding2         = compat_log.padding2;
                log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);

                r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
                break;
        }
        default:
                r = kvm_vm_ioctl(filp, ioctl, arg);
        }
        return r;
}
#endif

static struct file_operations kvm_vm_fops = {
        .release        = kvm_vm_release,
        .unlocked_ioctl = kvm_vm_ioctl,
        .llseek                = noop_llseek,
        KVM_COMPAT(kvm_vm_compat_ioctl),
};

bool file_is_kvm(struct file *file)
{
        return file && file->f_op == &kvm_vm_fops;
}
EXPORT_SYMBOL_GPL(file_is_kvm);

static int kvm_dev_ioctl_create_vm(unsigned long type)
{
        char fdname[ITOA_MAX_LEN + 1];
        int r, fd;
        struct kvm *kvm;
        struct file *file;

        fd = get_unused_fd_flags(O_CLOEXEC);
        if (fd < 0)
                return fd;

        snprintf(fdname, sizeof(fdname), "%d", fd);

        kvm = kvm_create_vm(type, fdname);
        if (IS_ERR(kvm)) {
                r = PTR_ERR(kvm);
                goto put_fd;
        }

        file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR);
        if (IS_ERR(file)) {
                r = PTR_ERR(file);
                goto put_kvm;
        }

        /*
         * Don't call kvm_put_kvm anymore at this point; file->f_op is
         * already set, with ->release() being kvm_vm_release().  In error
         * cases it will be called by the final fput(file) and will take
         * care of doing kvm_put_kvm(kvm).
         */
        kvm_uevent_notify_change(KVM_EVENT_CREATE_VM, kvm);

        fd_install(fd, file);
        return fd;

put_kvm:
        kvm_put_kvm(kvm);
put_fd:
        put_unused_fd(fd);
        return r;
}

static long kvm_dev_ioctl(struct file *filp,
                          unsigned int ioctl, unsigned long arg)
{
        int r = -EINVAL;

        switch (ioctl) {
        case KVM_GET_API_VERSION:
                if (arg)
                        goto out;
                r = KVM_API_VERSION;
                break;
        case KVM_CREATE_VM:
                r = kvm_dev_ioctl_create_vm(arg);
                break;
        case KVM_CHECK_EXTENSION:
                r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
                break;
        case KVM_GET_VCPU_MMAP_SIZE:
                if (arg)
                        goto out;
                r = PAGE_SIZE;     /* struct kvm_run */
#ifdef CONFIG_X86
                r += PAGE_SIZE;    /* pio data page */
#endif
#ifdef CONFIG_KVM_MMIO
                r += PAGE_SIZE;    /* coalesced mmio ring page */
#endif
                break;
        default:
                return kvm_arch_dev_ioctl(filp, ioctl, arg);
        }
out:
        return r;
}

static struct file_operations kvm_chardev_ops = {
        .unlocked_ioctl = kvm_dev_ioctl,
        .llseek                = noop_llseek,
        KVM_COMPAT(kvm_dev_ioctl),
};

static struct miscdevice kvm_dev = {
        KVM_MINOR,
        "kvm",
        &kvm_chardev_ops,
};

#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING
bool enable_virt_at_load = true;
module_param(enable_virt_at_load, bool, 0444);
EXPORT_SYMBOL_GPL(enable_virt_at_load);

__visible bool kvm_rebooting;
EXPORT_SYMBOL_GPL(kvm_rebooting);

static DEFINE_PER_CPU(bool, virtualization_enabled);
static DEFINE_MUTEX(kvm_usage_lock);
static int kvm_usage_count;

__weak void kvm_arch_enable_virtualization(void)
{

}

__weak void kvm_arch_disable_virtualization(void)
{

}

static int kvm_enable_virtualization_cpu(void)
{
        if (__this_cpu_read(virtualization_enabled))
                return 0;

        if (kvm_arch_enable_virtualization_cpu()) {
                pr_info("kvm: enabling virtualization on CPU%d failed\n",
                        raw_smp_processor_id());
                return -EIO;
        }

        __this_cpu_write(virtualization_enabled, true);
        return 0;
}

static int kvm_online_cpu(unsigned int cpu)
{
        /*
         * Abort the CPU online process if hardware virtualization cannot
         * be enabled. Otherwise running VMs would encounter unrecoverable
         * errors when scheduled to this CPU.
         */
        return kvm_enable_virtualization_cpu();
}

static void kvm_disable_virtualization_cpu(void *ign)
{
        if (!__this_cpu_read(virtualization_enabled))
                return;

        kvm_arch_disable_virtualization_cpu();

        __this_cpu_write(virtualization_enabled, false);
}

static int kvm_offline_cpu(unsigned int cpu)
{
        kvm_disable_virtualization_cpu(NULL);
        return 0;
}

static void kvm_shutdown(void)
{
        /*
         * Disable hardware virtualization and set kvm_rebooting to indicate
         * that KVM has asynchronously disabled hardware virtualization, i.e.
         * that relevant errors and exceptions aren't entirely unexpected.
         * Some flavors of hardware virtualization need to be disabled before
         * transferring control to firmware (to perform shutdown/reboot), e.g.
         * on x86, virtualization can block INIT interrupts, which are used by
         * firmware to pull APs back under firmware control.  Note, this path
         * is used for both shutdown and reboot scenarios, i.e. neither name is
         * 100% comprehensive.
         */
        pr_info("kvm: exiting hardware virtualization\n");
        kvm_rebooting = true;
        on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1);
}

static int kvm_suspend(void)
{
        /*
         * Secondary CPUs and CPU hotplug are disabled across the suspend/resume
         * callbacks, i.e. no need to acquire kvm_usage_lock to ensure the usage
         * count is stable.  Assert that kvm_usage_lock is not held to ensure
         * the system isn't suspended while KVM is enabling hardware.  Hardware
         * enabling can be preempted, but the task cannot be frozen until it has
         * dropped all locks (userspace tasks are frozen via a fake signal).
         */
        lockdep_assert_not_held(&kvm_usage_lock);
        lockdep_assert_irqs_disabled();

        kvm_disable_virtualization_cpu(NULL);
        return 0;
}

static void kvm_resume(void)
{
        lockdep_assert_not_held(&kvm_usage_lock);
        lockdep_assert_irqs_disabled();

        WARN_ON_ONCE(kvm_enable_virtualization_cpu());
}

static struct syscore_ops kvm_syscore_ops = {
        .suspend = kvm_suspend,
        .resume = kvm_resume,
        .shutdown = kvm_shutdown,
};

int kvm_enable_virtualization(void)
{
        int r;

        guard(mutex)(&kvm_usage_lock);

        if (kvm_usage_count++)
                return 0;

        kvm_arch_enable_virtualization();

        r = cpuhp_setup_state(CPUHP_AP_KVM_ONLINE, "kvm/cpu:online",
                              kvm_online_cpu, kvm_offline_cpu);
        if (r)
                goto err_cpuhp;

        register_syscore_ops(&kvm_syscore_ops);

        /*
         * Undo virtualization enabling and bail if the system is going down.
         * If userspace initiated a forced reboot, e.g. reboot -f, then it's
         * possible for an in-flight operation to enable virtualization after
         * syscore_shutdown() is called, i.e. without kvm_shutdown() being
         * invoked.  Note, this relies on system_state being set _before_
         * kvm_shutdown(), e.g. to ensure either kvm_shutdown() is invoked
         * or this CPU observes the impending shutdown.  Which is why KVM uses
         * a syscore ops hook instead of registering a dedicated reboot
         * notifier (the latter runs before system_state is updated).
         */
        if (system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF ||
            system_state == SYSTEM_RESTART) {
                r = -EBUSY;
                goto err_rebooting;
        }

        return 0;

err_rebooting:
        unregister_syscore_ops(&kvm_syscore_ops);
        cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
err_cpuhp:
        kvm_arch_disable_virtualization();
        --kvm_usage_count;
        return r;
}
EXPORT_SYMBOL_GPL(kvm_enable_virtualization);

void kvm_disable_virtualization(void)
{
        guard(mutex)(&kvm_usage_lock);

        if (--kvm_usage_count)
                return;

        unregister_syscore_ops(&kvm_syscore_ops);
        cpuhp_remove_state(CPUHP_AP_KVM_ONLINE);
        kvm_arch_disable_virtualization();
}
EXPORT_SYMBOL_GPL(kvm_disable_virtualization);

static int kvm_init_virtualization(void)
{
        if (enable_virt_at_load)
                return kvm_enable_virtualization();

        return 0;
}

static void kvm_uninit_virtualization(void)
{
        if (enable_virt_at_load)
                kvm_disable_virtualization();
}
#else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */
static int kvm_init_virtualization(void)
{
        return 0;
}

static void kvm_uninit_virtualization(void)
{

}
#endif /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */

static void kvm_iodevice_destructor(struct kvm_io_device *dev)
{
        if (dev->ops->destructor)
                dev->ops->destructor(dev);
}

static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
{
        int i;

        for (i = 0; i < bus->dev_count; i++) {
                struct kvm_io_device *pos = bus->range[i].dev;

                kvm_iodevice_destructor(pos);
        }
        kfree(bus);
}

static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
                                 const struct kvm_io_range *r2)
{
        gpa_t addr1 = r1->addr;
        gpa_t addr2 = r2->addr;

        if (addr1 < addr2)
                return -1;

        /* If r2->len == 0, match the exact address.  If r2->len != 0,
         * accept any overlapping write.  Any order is acceptable for
         * overlapping ranges, because kvm_io_bus_get_first_dev ensures
         * we process all of them.
         */
        if (r2->len) {
                addr1 += r1->len;
                addr2 += r2->len;
        }

        if (addr1 > addr2)
                return 1;

        return 0;
}

static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
{
        return kvm_io_bus_cmp(p1, p2);
}

static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
                             gpa_t addr, int len)
{
        struct kvm_io_range *range, key;
        int off;

        key = (struct kvm_io_range) {
                .addr = addr,
                .len = len,
        };

        range = bsearch(&key, bus->range, bus->dev_count,
                        sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
        if (range == NULL)
                return -ENOENT;

        off = range - bus->range;

        while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
                off--;

        return off;
}

static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
                              struct kvm_io_range *range, const void *val)
{
        int idx;

        idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
        if (idx < 0)
                return -EOPNOTSUPP;

        while (idx < bus->dev_count &&
                kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
                if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
                                        range->len, val))
                        return idx;
                idx++;
        }

        return -EOPNOTSUPP;
}

int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
                     int len, const void *val)
{
        struct kvm_io_bus *bus;
        struct kvm_io_range range;
        int r;

        range = (struct kvm_io_range) {
                .addr = addr,
                .len = len,
        };

        bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
        if (!bus)
                return -ENOMEM;
        r = __kvm_io_bus_write(vcpu, bus, &range, val);
        return r < 0 ? r : 0;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_write);

int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
                            gpa_t addr, int len, const void *val, long cookie)
{
        struct kvm_io_bus *bus;
        struct kvm_io_range range;

        range = (struct kvm_io_range) {
                .addr = addr,
                .len = len,
        };

        bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
        if (!bus)
                return -ENOMEM;

        /* First try the device referenced by cookie. */
        if ((cookie >= 0) && (cookie < bus->dev_count) &&
            (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
                if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
                                        val))
                        return cookie;

        /*
         * cookie contained garbage; fall back to search and return the
         * correct cookie value.
         */
        return __kvm_io_bus_write(vcpu, bus, &range, val);
}

static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
                             struct kvm_io_range *range, void *val)
{
        int idx;

        idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
        if (idx < 0)
                return -EOPNOTSUPP;

        while (idx < bus->dev_count &&
                kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
                if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
                                       range->len, val))
                        return idx;
                idx++;
        }

        return -EOPNOTSUPP;
}

int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
                    int len, void *val)
{
        struct kvm_io_bus *bus;
        struct kvm_io_range range;
        int r;

        range = (struct kvm_io_range) {
                .addr = addr,
                .len = len,
        };

        bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
        if (!bus)
                return -ENOMEM;
        r = __kvm_io_bus_read(vcpu, bus, &range, val);
        return r < 0 ? r : 0;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_read);

int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
                            int len, struct kvm_io_device *dev)
{
        int i;
        struct kvm_io_bus *new_bus, *bus;
        struct kvm_io_range range;

        lockdep_assert_held(&kvm->slots_lock);

        bus = kvm_get_bus(kvm, bus_idx);
        if (!bus)
                return -ENOMEM;

        /* exclude ioeventfd which is limited by maximum fd */
        if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
                return -ENOSPC;

        new_bus = kmalloc(struct_size(bus, range, bus->dev_count + 1),
                          GFP_KERNEL_ACCOUNT);
        if (!new_bus)
                return -ENOMEM;

        range = (struct kvm_io_range) {
                .addr = addr,
                .len = len,
                .dev = dev,
        };

        for (i = 0; i < bus->dev_count; i++)
                if (kvm_io_bus_cmp(&bus->range[i], &range) > 0)
                        break;

        memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
        new_bus->dev_count++;
        new_bus->range[i] = range;
        memcpy(new_bus->range + i + 1, bus->range + i,
                (bus->dev_count - i) * sizeof(struct kvm_io_range));
        rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
        synchronize_srcu_expedited(&kvm->srcu);
        kfree(bus);

        return 0;
}

int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
                              struct kvm_io_device *dev)
{
        int i;
        struct kvm_io_bus *new_bus, *bus;

        lockdep_assert_held(&kvm->slots_lock);

        bus = kvm_get_bus(kvm, bus_idx);
        if (!bus)
                return 0;

        for (i = 0; i < bus->dev_count; i++) {
                if (bus->range[i].dev == dev) {
                        break;
                }
        }

        if (i == bus->dev_count)
                return 0;

        new_bus = kmalloc(struct_size(bus, range, bus->dev_count - 1),
                          GFP_KERNEL_ACCOUNT);
        if (new_bus) {
                memcpy(new_bus, bus, struct_size(bus, range, i));
                new_bus->dev_count--;
                memcpy(new_bus->range + i, bus->range + i + 1,
                                flex_array_size(new_bus, range, new_bus->dev_count - i));
        }

        rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
        synchronize_srcu_expedited(&kvm->srcu);

        /*
         * If NULL bus is installed, destroy the old bus, including all the
         * attached devices. Otherwise, destroy the caller's device only.
         */
        if (!new_bus) {
                pr_err("kvm: failed to shrink bus, removing it completely\n");
                kvm_io_bus_destroy(bus);
                return -ENOMEM;
        }

        kvm_iodevice_destructor(dev);
        kfree(bus);
        return 0;
}

struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx,
                                         gpa_t addr)
{
        struct kvm_io_bus *bus;
        int dev_idx, srcu_idx;
        struct kvm_io_device *iodev = NULL;

        srcu_idx = srcu_read_lock(&kvm->srcu);

        bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
        if (!bus)
                goto out_unlock;

        dev_idx = kvm_io_bus_get_first_dev(bus, addr, 1);
        if (dev_idx < 0)
                goto out_unlock;

        iodev = bus->range[dev_idx].dev;

out_unlock:
        srcu_read_unlock(&kvm->srcu, srcu_idx);

        return iodev;
}
EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev);

static int kvm_debugfs_open(struct inode *inode, struct file *file,
                           int (*get)(void *, u64 *), int (*set)(void *, u64),
                           const char *fmt)
{
        int ret;
        struct kvm_stat_data *stat_data = inode->i_private;

        /*
         * The debugfs files are a reference to the kvm struct which
        * is still valid when kvm_destroy_vm is called.  kvm_get_kvm_safe
        * avoids the race between open and the removal of the debugfs directory.
         */
        if (!kvm_get_kvm_safe(stat_data->kvm))
                return -ENOENT;

        ret = simple_attr_open(inode, file, get,
                               kvm_stats_debugfs_mode(stat_data->desc) & 0222
                               ? set : NULL, fmt);
        if (ret)
                kvm_put_kvm(stat_data->kvm);

        return ret;
}

static int kvm_debugfs_release(struct inode *inode, struct file *file)
{
        struct kvm_stat_data *stat_data = inode->i_private;

        simple_attr_release(inode, file);
        kvm_put_kvm(stat_data->kvm);

        return 0;
}

static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val)
{
        *val = *(u64 *)((void *)(&kvm->stat) + offset);

        return 0;
}

static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset)
{
        *(u64 *)((void *)(&kvm->stat) + offset) = 0;

        return 0;
}

static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val)
{
        unsigned long i;
        struct kvm_vcpu *vcpu;

        *val = 0;

        kvm_for_each_vcpu(i, vcpu, kvm)
                *val += *(u64 *)((void *)(&vcpu->stat) + offset);

        return 0;
}

static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset)
{
        unsigned long i;
        struct kvm_vcpu *vcpu;

        kvm_for_each_vcpu(i, vcpu, kvm)
                *(u64 *)((void *)(&vcpu->stat) + offset) = 0;

        return 0;
}

static int kvm_stat_data_get(void *data, u64 *val)
{
        int r = -EFAULT;
        struct kvm_stat_data *stat_data = data;

        switch (stat_data->kind) {
        case KVM_STAT_VM:
                r = kvm_get_stat_per_vm(stat_data->kvm,
                                        stat_data->desc->desc.offset, val);
                break;
        case KVM_STAT_VCPU:
                r = kvm_get_stat_per_vcpu(stat_data->kvm,
                                          stat_data->desc->desc.offset, val);
                break;
        }

        return r;
}

static int kvm_stat_data_clear(void *data, u64 val)
{
        int r = -EFAULT;
        struct kvm_stat_data *stat_data = data;

        if (val)
                return -EINVAL;

        switch (stat_data->kind) {
        case KVM_STAT_VM:
                r = kvm_clear_stat_per_vm(stat_data->kvm,
                                          stat_data->desc->desc.offset);
                break;
        case KVM_STAT_VCPU:
                r = kvm_clear_stat_per_vcpu(stat_data->kvm,
                                            stat_data->desc->desc.offset);
                break;
        }

        return r;
}

static int kvm_stat_data_open(struct inode *inode, struct file *file)
{
        __simple_attr_check_format("%llu\n", 0ull);
        return kvm_debugfs_open(inode, file, kvm_stat_data_get,
                                kvm_stat_data_clear, "%llu\n");
}

static const struct file_operations stat_fops_per_vm = {
        .owner = THIS_MODULE,
        .open = kvm_stat_data_open,
        .release = kvm_debugfs_release,
        .read = simple_attr_read,
        .write = simple_attr_write,
};

static int vm_stat_get(void *_offset, u64 *val)
{
        unsigned offset = (long)_offset;
        struct kvm *kvm;
        u64 tmp_val;

        *val = 0;
        mutex_lock(&kvm_lock);
        list_for_each_entry(kvm, &vm_list, vm_list) {
                kvm_get_stat_per_vm(kvm, offset, &tmp_val);
                *val += tmp_val;
        }
        mutex_unlock(&kvm_lock);
        return 0;
}

static int vm_stat_clear(void *_offset, u64 val)
{
        unsigned offset = (long)_offset;
        struct kvm *kvm;

        if (val)
                return -EINVAL;

        mutex_lock(&kvm_lock);
        list_for_each_entry(kvm, &vm_list, vm_list) {
                kvm_clear_stat_per_vm(kvm, offset);
        }
        mutex_unlock(&kvm_lock);

        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n");

static int vcpu_stat_get(void *_offset, u64 *val)
{
        unsigned offset = (long)_offset;
        struct kvm *kvm;
        u64 tmp_val;

        *val = 0;
        mutex_lock(&kvm_lock);
        list_for_each_entry(kvm, &vm_list, vm_list) {
                kvm_get_stat_per_vcpu(kvm, offset, &tmp_val);
                *val += tmp_val;
        }
        mutex_unlock(&kvm_lock);
        return 0;
}

static int vcpu_stat_clear(void *_offset, u64 val)
{
        unsigned offset = (long)_offset;
        struct kvm *kvm;

        if (val)
                return -EINVAL;

        mutex_lock(&kvm_lock);
        list_for_each_entry(kvm, &vm_list, vm_list) {
                kvm_clear_stat_per_vcpu(kvm, offset);
        }
        mutex_unlock(&kvm_lock);

        return 0;
}

DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear,
                        "%llu\n");
DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n");

static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm)
{
        struct kobj_uevent_env *env;
        unsigned long long created, active;

        if (!kvm_dev.this_device || !kvm)
                return;

        mutex_lock(&kvm_lock);
        if (type == KVM_EVENT_CREATE_VM) {
                kvm_createvm_count++;
                kvm_active_vms++;
        } else if (type == KVM_EVENT_DESTROY_VM) {
                kvm_active_vms--;
        }
        created = kvm_createvm_count;
        active = kvm_active_vms;
        mutex_unlock(&kvm_lock);

        env = kzalloc(sizeof(*env), GFP_KERNEL);
        if (!env)
                return;

        add_uevent_var(env, "CREATED=%llu", created);
        add_uevent_var(env, "COUNT=%llu", active);

        if (type == KVM_EVENT_CREATE_VM) {
                add_uevent_var(env, "EVENT=create");
                kvm->userspace_pid = task_pid_nr(current);
        } else if (type == KVM_EVENT_DESTROY_VM) {
                add_uevent_var(env, "EVENT=destroy");
        }
        add_uevent_var(env, "PID=%d", kvm->userspace_pid);

        if (!IS_ERR(kvm->debugfs_dentry)) {
                char *tmp, *p = kmalloc(PATH_MAX, GFP_KERNEL);

                if (p) {
                        tmp = dentry_path_raw(kvm->debugfs_dentry, p, PATH_MAX);
                        if (!IS_ERR(tmp))
                                add_uevent_var(env, "STATS_PATH=%s", tmp);
                        kfree(p);
                }
        }
        /* no need for checks, since we are adding at most only 5 keys */
        env->envp[env->envp_idx++] = NULL;
        kobject_uevent_env(&kvm_dev.this_device->kobj, KOBJ_CHANGE, env->envp);
        kfree(env);
}

static void kvm_init_debug(void)
{
        const struct file_operations *fops;
        const struct _kvm_stats_desc *pdesc;
        int i;

        kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);

        for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) {
                pdesc = &kvm_vm_stats_desc[i];
                if (kvm_stats_debugfs_mode(pdesc) & 0222)
                        fops = &vm_stat_fops;
                else
                        fops = &vm_stat_readonly_fops;
                debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                                kvm_debugfs_dir,
                                (void *)(long)pdesc->desc.offset, fops);
        }

        for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) {
                pdesc = &kvm_vcpu_stats_desc[i];
                if (kvm_stats_debugfs_mode(pdesc) & 0222)
                        fops = &vcpu_stat_fops;
                else
                        fops = &vcpu_stat_readonly_fops;
                debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc),
                                kvm_debugfs_dir,
                                (void *)(long)pdesc->desc.offset, fops);
        }
}

static inline
struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
{
        return container_of(pn, struct kvm_vcpu, preempt_notifier);
}

static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
{
        struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

        WRITE_ONCE(vcpu->preempted, false);
        WRITE_ONCE(vcpu->ready, false);

        __this_cpu_write(kvm_running_vcpu, vcpu);
        kvm_arch_vcpu_load(vcpu, cpu);

        WRITE_ONCE(vcpu->scheduled_out, false);
}

static void kvm_sched_out(struct preempt_notifier *pn,
                          struct task_struct *next)
{
        struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);

        WRITE_ONCE(vcpu->scheduled_out, true);

        if (task_is_runnable(current) && vcpu->wants_to_run) {
                WRITE_ONCE(vcpu->preempted, true);
                WRITE_ONCE(vcpu->ready, true);
        }
        kvm_arch_vcpu_put(vcpu);
        __this_cpu_write(kvm_running_vcpu, NULL);
}

/**
 * kvm_get_running_vcpu - get the vcpu running on the current CPU.
 *
 * We can disable preemption locally around accessing the per-CPU variable,
 * and use the resolved vcpu pointer after enabling preemption again,
 * because even if the current thread is migrated to another CPU, reading
 * the per-CPU value later will give us the same value as we update the
 * per-CPU variable in the preempt notifier handlers.
 */
struct kvm_vcpu *kvm_get_running_vcpu(void)
{
        struct kvm_vcpu *vcpu;

        preempt_disable();
        vcpu = __this_cpu_read(kvm_running_vcpu);
        preempt_enable();

        return vcpu;
}
EXPORT_SYMBOL_GPL(kvm_get_running_vcpu);

/**
 * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus.
 */
struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void)
{
        return &kvm_running_vcpu;
}

#ifdef CONFIG_GUEST_PERF_EVENTS
static unsigned int kvm_guest_state(void)
{
        struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
        unsigned int state;

        if (!kvm_arch_pmi_in_guest(vcpu))
                return 0;

        state = PERF_GUEST_ACTIVE;
        if (!kvm_arch_vcpu_in_kernel(vcpu))
                state |= PERF_GUEST_USER;

        return state;
}

static unsigned long kvm_guest_get_ip(void)
{
        struct kvm_vcpu *vcpu = kvm_get_running_vcpu();

        /* Retrieving the IP must be guarded by a call to kvm_guest_state(). */
        if (WARN_ON_ONCE(!kvm_arch_pmi_in_guest(vcpu)))
                return 0;

        return kvm_arch_vcpu_get_ip(vcpu);
}

static struct perf_guest_info_callbacks kvm_guest_cbs = {
        .state                        = kvm_guest_state,
        .get_ip                        = kvm_guest_get_ip,
        .handle_intel_pt_intr        = NULL,
};

void kvm_register_perf_callbacks(unsigned int (*pt_intr_handler)(void))
{
        kvm_guest_cbs.handle_intel_pt_intr = pt_intr_handler;
        perf_register_guest_info_callbacks(&kvm_guest_cbs);
}
void kvm_unregister_perf_callbacks(void)
{
        perf_unregister_guest_info_callbacks(&kvm_guest_cbs);
}
#endif

int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
{
        int r;
        int cpu;

        /* A kmem cache lets us meet the alignment requirements of fx_save. */
        if (!vcpu_align)
                vcpu_align = __alignof__(struct kvm_vcpu);
        kvm_vcpu_cache =
                kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align,
                                           SLAB_ACCOUNT,
                                           offsetof(struct kvm_vcpu, arch),
                                           offsetofend(struct kvm_vcpu, stats_id)
                                           - offsetof(struct kvm_vcpu, arch),
                                           NULL);
        if (!kvm_vcpu_cache)
                return -ENOMEM;

        for_each_possible_cpu(cpu) {
                if (!alloc_cpumask_var_node(&per_cpu(cpu_kick_mask, cpu),
                                            GFP_KERNEL, cpu_to_node(cpu))) {
                        r = -ENOMEM;
                        goto err_cpu_kick_mask;
                }
        }

        r = kvm_irqfd_init();
        if (r)
                goto err_irqfd;

        r = kvm_async_pf_init();
        if (r)
                goto err_async_pf;

        kvm_chardev_ops.owner = module;
        kvm_vm_fops.owner = module;
        kvm_vcpu_fops.owner = module;
        kvm_device_fops.owner = module;

        kvm_preempt_ops.sched_in = kvm_sched_in;
        kvm_preempt_ops.sched_out = kvm_sched_out;

        kvm_init_debug();

        r = kvm_vfio_ops_init();
        if (WARN_ON_ONCE(r))
                goto err_vfio;

        kvm_gmem_init(module);

        r = kvm_init_virtualization();
        if (r)
                goto err_virt;

        /*
         * Registration _must_ be the very last thing done, as this exposes
         * /dev/kvm to userspace, i.e. all infrastructure must be setup!
         */
        r = misc_register(&kvm_dev);
        if (r) {
                pr_err("kvm: misc device register failed\n");
                goto err_register;
        }

        return 0;

err_register:
        kvm_uninit_virtualization();
err_virt:
        kvm_vfio_ops_exit();
err_vfio:
        kvm_async_pf_deinit();
err_async_pf:
        kvm_irqfd_exit();
err_irqfd:
err_cpu_kick_mask:
        for_each_possible_cpu(cpu)
                free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
        kmem_cache_destroy(kvm_vcpu_cache);
        return r;
}
EXPORT_SYMBOL_GPL(kvm_init);

void kvm_exit(void)
{
        int cpu;

        /*
         * Note, unregistering /dev/kvm doesn't strictly need to come first,
         * fops_get(), a.k.a. try_module_get(), prevents acquiring references
         * to KVM while the module is being stopped.
         */
        misc_deregister(&kvm_dev);

        kvm_uninit_virtualization();

        debugfs_remove_recursive(kvm_debugfs_dir);
        for_each_possible_cpu(cpu)
                free_cpumask_var(per_cpu(cpu_kick_mask, cpu));
        kmem_cache_destroy(kvm_vcpu_cache);
        kvm_vfio_ops_exit();
        kvm_async_pf_deinit();
        kvm_irqfd_exit();
}
EXPORT_SYMBOL_GPL(kvm_exit);















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    6 

    6 

    6 


















    6 











































































































































































































































































































































































































































    6 
    6 









    6 
    6 































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        NET3        IP device support routines.
 *
 *        Derived from the IP parts of dev.c 1.0.19
 *                 Authors:        Ross Biro
 *                                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                                Mark Evans, <evansmp@uhura.aston.ac.uk>
 *
 *        Additional Authors:
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *                Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 *        Changes:
 *                Alexey Kuznetsov:        pa_* fields are replaced with ifaddr
 *                                        lists.
 *                Cyrus Durgin:                updated for kmod
 *                Matthias Andree:        in devinet_ioctl, compare label and
 *                                        address (4.4BSD alias style support),
 *                                        fall back to comparing just the label
 *                                        if no match found.
 */


#include <linux/uaccess.h>
#include <linux/bitops.h>
#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/in.h>
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/if_addr.h>
#include <linux/if_ether.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/init.h>
#include <linux/notifier.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include "igmp_internal.h"
#include <linux/slab.h>
#include <linux/hash.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif
#include <linux/kmod.h>
#include <linux/netconf.h>

#include <net/arp.h>
#include <net/ip.h>
#include <net/route.h>
#include <net/ip_fib.h>
#include <net/rtnetlink.h>
#include <net/net_namespace.h>
#include <net/addrconf.h>

#define IPV6ONLY_FLAGS        \
                (IFA_F_NODAD | IFA_F_OPTIMISTIC | IFA_F_DADFAILED | \
                 IFA_F_HOMEADDRESS | IFA_F_TENTATIVE | \
                 IFA_F_MANAGETEMPADDR | IFA_F_STABLE_PRIVACY)

static struct ipv4_devconf ipv4_devconf = {
        .data = {
                [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
                [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL - 1] = 10000 /*ms*/,
                [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL - 1] =  1000 /*ms*/,
                [IPV4_DEVCONF_ARP_EVICT_NOCARRIER - 1] = 1,
        },
};

static struct ipv4_devconf ipv4_devconf_dflt = {
        .data = {
                [IPV4_DEVCONF_ACCEPT_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SEND_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SECURE_REDIRECTS - 1] = 1,
                [IPV4_DEVCONF_SHARED_MEDIA - 1] = 1,
                [IPV4_DEVCONF_ACCEPT_SOURCE_ROUTE - 1] = 1,
                [IPV4_DEVCONF_IGMPV2_UNSOLICITED_REPORT_INTERVAL - 1] = 10000 /*ms*/,
                [IPV4_DEVCONF_IGMPV3_UNSOLICITED_REPORT_INTERVAL - 1] =  1000 /*ms*/,
                [IPV4_DEVCONF_ARP_EVICT_NOCARRIER - 1] = 1,
        },
};

#define IPV4_DEVCONF_DFLT(net, attr) \
        IPV4_DEVCONF((*net->ipv4.devconf_dflt), attr)

static const struct nla_policy ifa_ipv4_policy[IFA_MAX+1] = {
        [IFA_LOCAL]             = { .type = NLA_U32 },
        [IFA_ADDRESS]           = { .type = NLA_U32 },
        [IFA_BROADCAST]         = { .type = NLA_U32 },
        [IFA_LABEL]             = { .type = NLA_STRING, .len = IFNAMSIZ - 1 },
        [IFA_CACHEINFO]                = { .len = sizeof(struct ifa_cacheinfo) },
        [IFA_FLAGS]                = { .type = NLA_U32 },
        [IFA_RT_PRIORITY]        = { .type = NLA_U32 },
        [IFA_TARGET_NETNSID]        = { .type = NLA_S32 },
        [IFA_PROTO]                = { .type = NLA_U8 },
};

#define IN4_ADDR_HSIZE_SHIFT        8
#define IN4_ADDR_HSIZE                (1U << IN4_ADDR_HSIZE_SHIFT)

static u32 inet_addr_hash(const struct net *net, __be32 addr)
{
        u32 val = __ipv4_addr_hash(addr, net_hash_mix(net));

        return hash_32(val, IN4_ADDR_HSIZE_SHIFT);
}

static void inet_hash_insert(struct net *net, struct in_ifaddr *ifa)
{
        u32 hash = inet_addr_hash(net, ifa->ifa_local);

        ASSERT_RTNL();
        hlist_add_head_rcu(&ifa->addr_lst, &net->ipv4.inet_addr_lst[hash]);
}

static void inet_hash_remove(struct in_ifaddr *ifa)
{
        ASSERT_RTNL();
        hlist_del_init_rcu(&ifa->addr_lst);
}

/**
 * __ip_dev_find - find the first device with a given source address.
 * @net: the net namespace
 * @addr: the source address
 * @devref: if true, take a reference on the found device
 *
 * If a caller uses devref=false, it should be protected by RCU, or RTNL
 */
struct net_device *__ip_dev_find(struct net *net, __be32 addr, bool devref)
{
        struct net_device *result = NULL;
        struct in_ifaddr *ifa;

        rcu_read_lock();
        ifa = inet_lookup_ifaddr_rcu(net, addr);
        if (!ifa) {
                struct flowi4 fl4 = { .daddr = addr };
                struct fib_result res = { 0 };
                struct fib_table *local;

                /* Fallback to FIB local table so that communication
                 * over loopback subnets work.
                 */
                local = fib_get_table(net, RT_TABLE_LOCAL);
                if (local &&
                    !fib_table_lookup(local, &fl4, &res, FIB_LOOKUP_NOREF) &&
                    res.type == RTN_LOCAL)
                        result = FIB_RES_DEV(res);
        } else {
                result = ifa->ifa_dev->dev;
        }
        if (result && devref)
                dev_hold(result);
        rcu_read_unlock();
        return result;
}
EXPORT_SYMBOL(__ip_dev_find);

/* called under RCU lock */
struct in_ifaddr *inet_lookup_ifaddr_rcu(struct net *net, __be32 addr)
{
        u32 hash = inet_addr_hash(net, addr);
        struct in_ifaddr *ifa;

        hlist_for_each_entry_rcu(ifa, &net->ipv4.inet_addr_lst[hash], addr_lst)
                if (ifa->ifa_local == addr)
                        return ifa;

        return NULL;
}

static void rtmsg_ifa(int event, struct in_ifaddr *, struct nlmsghdr *, u32);

static BLOCKING_NOTIFIER_HEAD(inetaddr_chain);
static BLOCKING_NOTIFIER_HEAD(inetaddr_validator_chain);
static void inet_del_ifa(struct in_device *in_dev,
                         struct in_ifaddr __rcu **ifap,
                         int destroy);
#ifdef CONFIG_SYSCTL
static int devinet_sysctl_register(struct in_device *idev);
static void devinet_sysctl_unregister(struct in_device *idev);
#else
static int devinet_sysctl_register(struct in_device *idev)
{
        return 0;
}
static void devinet_sysctl_unregister(struct in_device *idev)
{
}
#endif

/* Locks all the inet devices. */

static struct in_ifaddr *inet_alloc_ifa(struct in_device *in_dev)
{
        struct in_ifaddr *ifa;

        ifa = kzalloc(sizeof(*ifa), GFP_KERNEL_ACCOUNT);
        if (!ifa)
                return NULL;

        in_dev_hold(in_dev);
        ifa->ifa_dev = in_dev;

        INIT_HLIST_NODE(&ifa->addr_lst);

        return ifa;
}

static void inet_rcu_free_ifa(struct rcu_head *head)
{
        struct in_ifaddr *ifa = container_of(head, struct in_ifaddr, rcu_head);

        in_dev_put(ifa->ifa_dev);
        kfree(ifa);
}

static void inet_free_ifa(struct in_ifaddr *ifa)
{
        /* Our reference to ifa->ifa_dev must be freed ASAP
         * to release the reference to the netdev the same way.
         * in_dev_put() -> in_dev_finish_destroy() -> netdev_put()
         */
        call_rcu_hurry(&ifa->rcu_head, inet_rcu_free_ifa);
}

static void in_dev_free_rcu(struct rcu_head *head)
{
        struct in_device *idev = container_of(head, struct in_device, rcu_head);

        kfree(rcu_dereference_protected(idev->mc_hash, 1));
        kfree(idev);
}

void in_dev_finish_destroy(struct in_device *idev)
{
        struct net_device *dev = idev->dev;

        WARN_ON(idev->ifa_list);
        WARN_ON(idev->mc_list);
#ifdef NET_REFCNT_DEBUG
        pr_debug("%s: %p=%s\n", __func__, idev, dev ? dev->name : "NIL");
#endif
        netdev_put(dev, &idev->dev_tracker);
        if (!idev->dead)
                pr_err("Freeing alive in_device %p\n", idev);
        else
                call_rcu(&idev->rcu_head, in_dev_free_rcu);
}
EXPORT_SYMBOL(in_dev_finish_destroy);

static struct in_device *inetdev_init(struct net_device *dev)
{
        struct in_device *in_dev;
        int err = -ENOMEM;

        ASSERT_RTNL();

        in_dev = kzalloc(sizeof(*in_dev), GFP_KERNEL);
        if (!in_dev)
                goto out;
        memcpy(&in_dev->cnf, dev_net(dev)->ipv4.devconf_dflt,
                        sizeof(in_dev->cnf));
        in_dev->cnf.sysctl = NULL;
        in_dev->dev = dev;
        in_dev->arp_parms = neigh_parms_alloc(dev, &arp_tbl);
        if (!in_dev->arp_parms)
                goto out_kfree;
        if (IPV4_DEVCONF(in_dev->cnf, FORWARDING))
                netif_disable_lro(dev);
        /* Reference in_dev->dev */
        netdev_hold(dev, &in_dev->dev_tracker, GFP_KERNEL);
        /* Account for reference dev->ip_ptr (below) */
        refcount_set(&in_dev->refcnt, 1);

        if (dev != blackhole_netdev) {
                err = devinet_sysctl_register(in_dev);
                if (err) {
                        in_dev->dead = 1;
                        neigh_parms_release(&arp_tbl, in_dev->arp_parms);
                        in_dev_put(in_dev);
                        in_dev = NULL;
                        goto out;
                }
                ip_mc_init_dev(in_dev);
                if (dev->flags & IFF_UP)
                        ip_mc_up(in_dev);
        }

        /* we can receive as soon as ip_ptr is set -- do this last */
        rcu_assign_pointer(dev->ip_ptr, in_dev);
out:
        return in_dev ?: ERR_PTR(err);
out_kfree:
        kfree(in_dev);
        in_dev = NULL;
        goto out;
}

static void inetdev_destroy(struct in_device *in_dev)
{
        struct net_device *dev;
        struct in_ifaddr *ifa;

        ASSERT_RTNL();

        dev = in_dev->dev;

        in_dev->dead = 1;

        ip_mc_destroy_dev(in_dev);

        while ((ifa = rtnl_dereference(in_dev->ifa_list)) != NULL) {
                inet_del_ifa(in_dev, &in_dev->ifa_list, 0);
                inet_free_ifa(ifa);
        }

        RCU_INIT_POINTER(dev->ip_ptr, NULL);

        devinet_sysctl_unregister(in_dev);
        neigh_parms_release(&arp_tbl, in_dev->arp_parms);
        arp_ifdown(dev);

        in_dev_put(in_dev);
}

static int __init inet_blackhole_dev_init(void)
{
        int err = 0;

        rtnl_lock();
        if (!inetdev_init(blackhole_netdev))
                err = -ENOMEM;
        rtnl_unlock();

        return err;
}
late_initcall(inet_blackhole_dev_init);

int inet_addr_onlink(struct in_device *in_dev, __be32 a, __be32 b)
{
        const struct in_ifaddr *ifa;

        rcu_read_lock();
        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (inet_ifa_match(a, ifa)) {
                        if (!b || inet_ifa_match(b, ifa)) {
                                rcu_read_unlock();
                                return 1;
                        }
                }
        }
        rcu_read_unlock();
        return 0;
}

static void __inet_del_ifa(struct in_device *in_dev,
                           struct in_ifaddr __rcu **ifap,
                           int destroy, struct nlmsghdr *nlh, u32 portid)
{
        struct in_ifaddr *promote = NULL;
        struct in_ifaddr *ifa, *ifa1;
        struct in_ifaddr __rcu **last_prim;
        struct in_ifaddr *prev_prom = NULL;
        int do_promote = IN_DEV_PROMOTE_SECONDARIES(in_dev);

        ASSERT_RTNL();

        ifa1 = rtnl_dereference(*ifap);
        last_prim = ifap;
        if (in_dev->dead)
                goto no_promotions;

        /* 1. Deleting primary ifaddr forces deletion all secondaries
         * unless alias promotion is set
         **/

        if (!(ifa1->ifa_flags & IFA_F_SECONDARY)) {
                struct in_ifaddr __rcu **ifap1 = &ifa1->ifa_next;

                while ((ifa = rtnl_dereference(*ifap1)) != NULL) {
                        if (!(ifa->ifa_flags & IFA_F_SECONDARY) &&
                            ifa1->ifa_scope <= ifa->ifa_scope)
                                last_prim = &ifa->ifa_next;

                        if (!(ifa->ifa_flags & IFA_F_SECONDARY) ||
                            ifa1->ifa_mask != ifa->ifa_mask ||
                            !inet_ifa_match(ifa1->ifa_address, ifa)) {
                                ifap1 = &ifa->ifa_next;
                                prev_prom = ifa;
                                continue;
                        }

                        if (!do_promote) {
                                inet_hash_remove(ifa);
                                *ifap1 = ifa->ifa_next;

                                rtmsg_ifa(RTM_DELADDR, ifa, nlh, portid);
                                blocking_notifier_call_chain(&inetaddr_chain,
                                                NETDEV_DOWN, ifa);
                                inet_free_ifa(ifa);
                        } else {
                                promote = ifa;
                                break;
                        }
                }
        }

        /* On promotion all secondaries from subnet are changing
         * the primary IP, we must remove all their routes silently
         * and later to add them back with new prefsrc. Do this
         * while all addresses are on the device list.
         */
        for (ifa = promote; ifa; ifa = rtnl_dereference(ifa->ifa_next)) {
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa))
                        fib_del_ifaddr(ifa, ifa1);
        }

no_promotions:
        /* 2. Unlink it */

        *ifap = ifa1->ifa_next;
        inet_hash_remove(ifa1);

        /* 3. Announce address deletion */

        /* Send message first, then call notifier.
           At first sight, FIB update triggered by notifier
           will refer to already deleted ifaddr, that could confuse
           netlink listeners. It is not true: look, gated sees
           that route deleted and if it still thinks that ifaddr
           is valid, it will try to restore deleted routes... Grr.
           So that, this order is correct.
         */
        rtmsg_ifa(RTM_DELADDR, ifa1, nlh, portid);
        blocking_notifier_call_chain(&inetaddr_chain, NETDEV_DOWN, ifa1);

        if (promote) {
                struct in_ifaddr *next_sec;

                next_sec = rtnl_dereference(promote->ifa_next);
                if (prev_prom) {
                        struct in_ifaddr *last_sec;

                        rcu_assign_pointer(prev_prom->ifa_next, next_sec);

                        last_sec = rtnl_dereference(*last_prim);
                        rcu_assign_pointer(promote->ifa_next, last_sec);
                        rcu_assign_pointer(*last_prim, promote);
                }

                promote->ifa_flags &= ~IFA_F_SECONDARY;
                rtmsg_ifa(RTM_NEWADDR, promote, nlh, portid);
                blocking_notifier_call_chain(&inetaddr_chain,
                                NETDEV_UP, promote);
                for (ifa = next_sec; ifa;
                     ifa = rtnl_dereference(ifa->ifa_next)) {
                        if (ifa1->ifa_mask != ifa->ifa_mask ||
                            !inet_ifa_match(ifa1->ifa_address, ifa))
                                        continue;
                        fib_add_ifaddr(ifa);
                }

        }
        if (destroy)
                inet_free_ifa(ifa1);
}

static void inet_del_ifa(struct in_device *in_dev,
                         struct in_ifaddr __rcu **ifap,
                         int destroy)
{
        __inet_del_ifa(in_dev, ifap, destroy, NULL, 0);
}

static int __inet_insert_ifa(struct in_ifaddr *ifa, struct nlmsghdr *nlh,
                             u32 portid, struct netlink_ext_ack *extack)
{
        struct in_ifaddr __rcu **last_primary, **ifap;
        struct in_device *in_dev = ifa->ifa_dev;
        struct net *net = dev_net(in_dev->dev);
        struct in_validator_info ivi;
        struct in_ifaddr *ifa1;
        int ret;

        ASSERT_RTNL();

        ifa->ifa_flags &= ~IFA_F_SECONDARY;
        last_primary = &in_dev->ifa_list;

        /* Don't set IPv6 only flags to IPv4 addresses */
        ifa->ifa_flags &= ~IPV6ONLY_FLAGS;

        ifap = &in_dev->ifa_list;
        ifa1 = rtnl_dereference(*ifap);

        while (ifa1) {
                if (!(ifa1->ifa_flags & IFA_F_SECONDARY) &&
                    ifa->ifa_scope <= ifa1->ifa_scope)
                        last_primary = &ifa1->ifa_next;
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa)) {
                        if (ifa1->ifa_local == ifa->ifa_local) {
                                inet_free_ifa(ifa);
                                return -EEXIST;
                        }
                        if (ifa1->ifa_scope != ifa->ifa_scope) {
                                NL_SET_ERR_MSG(extack, "ipv4: Invalid scope value");
                                inet_free_ifa(ifa);
                                return -EINVAL;
                        }
                        ifa->ifa_flags |= IFA_F_SECONDARY;
                }

                ifap = &ifa1->ifa_next;
                ifa1 = rtnl_dereference(*ifap);
        }

        /* Allow any devices that wish to register ifaddr validtors to weigh
         * in now, before changes are committed.  The rntl lock is serializing
         * access here, so the state should not change between a validator call
         * and a final notify on commit.  This isn't invoked on promotion under
         * the assumption that validators are checking the address itself, and
         * not the flags.
         */
        ivi.ivi_addr = ifa->ifa_address;
        ivi.ivi_dev = ifa->ifa_dev;
        ivi.extack = extack;
        ret = blocking_notifier_call_chain(&inetaddr_validator_chain,
                                           NETDEV_UP, &ivi);
        ret = notifier_to_errno(ret);
        if (ret) {
                inet_free_ifa(ifa);
                return ret;
        }

        if (!(ifa->ifa_flags & IFA_F_SECONDARY))
                ifap = last_primary;

        rcu_assign_pointer(ifa->ifa_next, *ifap);
        rcu_assign_pointer(*ifap, ifa);

        inet_hash_insert(dev_net(in_dev->dev), ifa);

        cancel_delayed_work(&net->ipv4.addr_chk_work);
        queue_delayed_work(system_power_efficient_wq, &net->ipv4.addr_chk_work, 0);

        /* Send message first, then call notifier.
           Notifier will trigger FIB update, so that
           listeners of netlink will know about new ifaddr */
        rtmsg_ifa(RTM_NEWADDR, ifa, nlh, portid);
        blocking_notifier_call_chain(&inetaddr_chain, NETDEV_UP, ifa);

        return 0;
}

static int inet_insert_ifa(struct in_ifaddr *ifa)
{
        if (!ifa->ifa_local) {
                inet_free_ifa(ifa);
                return 0;
        }

        return __inet_insert_ifa(ifa, NULL, 0, NULL);
}

static int inet_set_ifa(struct net_device *dev, struct in_ifaddr *ifa)
{
        struct in_device *in_dev = __in_dev_get_rtnl_net(dev);

        ipv4_devconf_setall(in_dev);
        neigh_parms_data_state_setall(in_dev->arp_parms);

        if (ipv4_is_loopback(ifa->ifa_local))
                ifa->ifa_scope = RT_SCOPE_HOST;
        return inet_insert_ifa(ifa);
}

/* Caller must hold RCU or RTNL :
 * We dont take a reference on found in_device
 */
struct in_device *inetdev_by_index(struct net *net, int ifindex)
{
        struct net_device *dev;
        struct in_device *in_dev = NULL;

        rcu_read_lock();
        dev = dev_get_by_index_rcu(net, ifindex);
        if (dev)
                in_dev = rcu_dereference_rtnl(dev->ip_ptr);
        rcu_read_unlock();
        return in_dev;
}
EXPORT_SYMBOL(inetdev_by_index);

/* Called only from RTNL semaphored context. No locks. */

struct in_ifaddr *inet_ifa_byprefix(struct in_device *in_dev, __be32 prefix,
                                    __be32 mask)
{
        struct in_ifaddr *ifa;

        ASSERT_RTNL();

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                if (ifa->ifa_mask == mask && inet_ifa_match(prefix, ifa))
                        return ifa;
        }
        return NULL;
}

static int ip_mc_autojoin_config(struct net *net, bool join,
                                 const struct in_ifaddr *ifa)
{
#if defined(CONFIG_IP_MULTICAST)
        struct ip_mreqn mreq = {
                .imr_multiaddr.s_addr = ifa->ifa_address,
                .imr_ifindex = ifa->ifa_dev->dev->ifindex,
        };
        struct sock *sk = net->ipv4.mc_autojoin_sk;
        int ret;

        ASSERT_RTNL_NET(net);

        lock_sock(sk);
        if (join)
                ret = ip_mc_join_group(sk, &mreq);
        else
                ret = ip_mc_leave_group(sk, &mreq);
        release_sock(sk);

        return ret;
#else
        return -EOPNOTSUPP;
#endif
}

static int inet_rtm_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct in_ifaddr __rcu **ifap;
        struct nlattr *tb[IFA_MAX+1];
        struct in_device *in_dev;
        struct ifaddrmsg *ifm;
        struct in_ifaddr *ifa;
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
                                     ifa_ipv4_policy, extack);
        if (err < 0)
                goto out;

        ifm = nlmsg_data(nlh);

        rtnl_net_lock(net);

        in_dev = inetdev_by_index(net, ifm->ifa_index);
        if (!in_dev) {
                NL_SET_ERR_MSG(extack, "ipv4: Device not found");
                err = -ENODEV;
                goto unlock;
        }

        for (ifap = &in_dev->ifa_list;
             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
             ifap = &ifa->ifa_next) {
                if (tb[IFA_LOCAL] &&
                    ifa->ifa_local != nla_get_in_addr(tb[IFA_LOCAL]))
                        continue;

                if (tb[IFA_LABEL] && nla_strcmp(tb[IFA_LABEL], ifa->ifa_label))
                        continue;

                if (tb[IFA_ADDRESS] &&
                    (ifm->ifa_prefixlen != ifa->ifa_prefixlen ||
                    !inet_ifa_match(nla_get_in_addr(tb[IFA_ADDRESS]), ifa)))
                        continue;

                if (ipv4_is_multicast(ifa->ifa_address))
                        ip_mc_autojoin_config(net, false, ifa);

                __inet_del_ifa(in_dev, ifap, 1, nlh, NETLINK_CB(skb).portid);
                goto unlock;
        }

        NL_SET_ERR_MSG(extack, "ipv4: Address not found");
        err = -EADDRNOTAVAIL;
unlock:
        rtnl_net_unlock(net);
out:
        return err;
}

static void check_lifetime(struct work_struct *work)
{
        unsigned long now, next, next_sec, next_sched;
        struct in_ifaddr *ifa;
        struct hlist_node *n;
        struct net *net;
        int i;

        net = container_of(to_delayed_work(work), struct net, ipv4.addr_chk_work);
        now = jiffies;
        next = round_jiffies_up(now + ADDR_CHECK_FREQUENCY);

        for (i = 0; i < IN4_ADDR_HSIZE; i++) {
                struct hlist_head *head = &net->ipv4.inet_addr_lst[i];
                bool change_needed = false;

                rcu_read_lock();
                hlist_for_each_entry_rcu(ifa, head, addr_lst) {
                        unsigned long age, tstamp;
                        u32 preferred_lft;
                        u32 valid_lft;
                        u32 flags;

                        flags = READ_ONCE(ifa->ifa_flags);
                        if (flags & IFA_F_PERMANENT)
                                continue;

                        preferred_lft = READ_ONCE(ifa->ifa_preferred_lft);
                        valid_lft = READ_ONCE(ifa->ifa_valid_lft);
                        tstamp = READ_ONCE(ifa->ifa_tstamp);
                        /* We try to batch several events at once. */
                        age = (now - tstamp +
                               ADDRCONF_TIMER_FUZZ_MINUS) / HZ;

                        if (valid_lft != INFINITY_LIFE_TIME &&
                            age >= valid_lft) {
                                change_needed = true;
                        } else if (preferred_lft ==
                                   INFINITY_LIFE_TIME) {
                                continue;
                        } else if (age >= preferred_lft) {
                                if (time_before(tstamp + valid_lft * HZ, next))
                                        next = tstamp + valid_lft * HZ;

                                if (!(flags & IFA_F_DEPRECATED))
                                        change_needed = true;
                        } else if (time_before(tstamp + preferred_lft * HZ,
                                               next)) {
                                next = tstamp + preferred_lft * HZ;
                        }
                }
                rcu_read_unlock();
                if (!change_needed)
                        continue;

                rtnl_net_lock(net);
                hlist_for_each_entry_safe(ifa, n, head, addr_lst) {
                        unsigned long age;

                        if (ifa->ifa_flags & IFA_F_PERMANENT)
                                continue;

                        /* We try to batch several events at once. */
                        age = (now - ifa->ifa_tstamp +
                               ADDRCONF_TIMER_FUZZ_MINUS) / HZ;

                        if (ifa->ifa_valid_lft != INFINITY_LIFE_TIME &&
                            age >= ifa->ifa_valid_lft) {
                                struct in_ifaddr __rcu **ifap;
                                struct in_ifaddr *tmp;

                                ifap = &ifa->ifa_dev->ifa_list;
                                tmp = rtnl_net_dereference(net, *ifap);
                                while (tmp) {
                                        if (tmp == ifa) {
                                                inet_del_ifa(ifa->ifa_dev,
                                                             ifap, 1);
                                                break;
                                        }
                                        ifap = &tmp->ifa_next;
                                        tmp = rtnl_net_dereference(net, *ifap);
                                }
                        } else if (ifa->ifa_preferred_lft !=
                                   INFINITY_LIFE_TIME &&
                                   age >= ifa->ifa_preferred_lft &&
                                   !(ifa->ifa_flags & IFA_F_DEPRECATED)) {
                                ifa->ifa_flags |= IFA_F_DEPRECATED;
                                rtmsg_ifa(RTM_NEWADDR, ifa, NULL, 0);
                        }
                }
                rtnl_net_unlock(net);
        }

        next_sec = round_jiffies_up(next);
        next_sched = next;

        /* If rounded timeout is accurate enough, accept it. */
        if (time_before(next_sec, next + ADDRCONF_TIMER_FUZZ))
                next_sched = next_sec;

        now = jiffies;
        /* And minimum interval is ADDRCONF_TIMER_FUZZ_MAX. */
        if (time_before(next_sched, now + ADDRCONF_TIMER_FUZZ_MAX))
                next_sched = now + ADDRCONF_TIMER_FUZZ_MAX;

        queue_delayed_work(system_power_efficient_wq, &net->ipv4.addr_chk_work,
                           next_sched - now);
}

static void set_ifa_lifetime(struct in_ifaddr *ifa, __u32 valid_lft,
                             __u32 prefered_lft)
{
        unsigned long timeout;
        u32 flags;

        flags = ifa->ifa_flags & ~(IFA_F_PERMANENT | IFA_F_DEPRECATED);

        timeout = addrconf_timeout_fixup(valid_lft, HZ);
        if (addrconf_finite_timeout(timeout))
                WRITE_ONCE(ifa->ifa_valid_lft, timeout);
        else
                flags |= IFA_F_PERMANENT;

        timeout = addrconf_timeout_fixup(prefered_lft, HZ);
        if (addrconf_finite_timeout(timeout)) {
                if (timeout == 0)
                        flags |= IFA_F_DEPRECATED;
                WRITE_ONCE(ifa->ifa_preferred_lft, timeout);
        }
        WRITE_ONCE(ifa->ifa_flags, flags);
        WRITE_ONCE(ifa->ifa_tstamp, jiffies);
        if (!ifa->ifa_cstamp)
                WRITE_ONCE(ifa->ifa_cstamp, ifa->ifa_tstamp);
}

static int inet_validate_rtm(struct nlmsghdr *nlh, struct nlattr **tb,
                             struct netlink_ext_ack *extack,
                             __u32 *valid_lft, __u32 *prefered_lft)
{
        struct ifaddrmsg *ifm = nlmsg_data(nlh);
        int err;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
                                     ifa_ipv4_policy, extack);
        if (err < 0)
                return err;

        if (ifm->ifa_prefixlen > 32) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid prefix length");
                return -EINVAL;
        }

        if (!tb[IFA_LOCAL]) {
                NL_SET_ERR_MSG(extack, "ipv4: Local address is not supplied");
                return -EINVAL;
        }

        if (tb[IFA_CACHEINFO]) {
                struct ifa_cacheinfo *ci;

                ci = nla_data(tb[IFA_CACHEINFO]);
                if (!ci->ifa_valid || ci->ifa_prefered > ci->ifa_valid) {
                        NL_SET_ERR_MSG(extack, "ipv4: address lifetime invalid");
                        return -EINVAL;
                }

                *valid_lft = ci->ifa_valid;
                *prefered_lft = ci->ifa_prefered;
        }

        return 0;
}

static struct in_ifaddr *inet_rtm_to_ifa(struct net *net, struct nlmsghdr *nlh,
                                         struct nlattr **tb,
                                         struct netlink_ext_ack *extack)
{
        struct ifaddrmsg *ifm = nlmsg_data(nlh);
        struct in_device *in_dev;
        struct net_device *dev;
        struct in_ifaddr *ifa;
        int err;

        dev = __dev_get_by_index(net, ifm->ifa_index);
        err = -ENODEV;
        if (!dev) {
                NL_SET_ERR_MSG(extack, "ipv4: Device not found");
                goto errout;
        }

        in_dev = __in_dev_get_rtnl_net(dev);
        err = -ENOBUFS;
        if (!in_dev)
                goto errout;

        ifa = inet_alloc_ifa(in_dev);
        if (!ifa)
                /*
                 * A potential indev allocation can be left alive, it stays
                 * assigned to its device and is destroy with it.
                 */
                goto errout;

        ipv4_devconf_setall(in_dev);
        neigh_parms_data_state_setall(in_dev->arp_parms);

        if (!tb[IFA_ADDRESS])
                tb[IFA_ADDRESS] = tb[IFA_LOCAL];

        ifa->ifa_prefixlen = ifm->ifa_prefixlen;
        ifa->ifa_mask = inet_make_mask(ifm->ifa_prefixlen);
        ifa->ifa_flags = nla_get_u32_default(tb[IFA_FLAGS], ifm->ifa_flags);
        ifa->ifa_scope = ifm->ifa_scope;
        ifa->ifa_local = nla_get_in_addr(tb[IFA_LOCAL]);
        ifa->ifa_address = nla_get_in_addr(tb[IFA_ADDRESS]);

        if (tb[IFA_BROADCAST])
                ifa->ifa_broadcast = nla_get_in_addr(tb[IFA_BROADCAST]);

        if (tb[IFA_LABEL])
                nla_strscpy(ifa->ifa_label, tb[IFA_LABEL], IFNAMSIZ);
        else
                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);

        if (tb[IFA_RT_PRIORITY])
                ifa->ifa_rt_priority = nla_get_u32(tb[IFA_RT_PRIORITY]);

        if (tb[IFA_PROTO])
                ifa->ifa_proto = nla_get_u8(tb[IFA_PROTO]);

        return ifa;

errout:
        return ERR_PTR(err);
}

static struct in_ifaddr *find_matching_ifa(struct net *net, struct in_ifaddr *ifa)
{
        struct in_device *in_dev = ifa->ifa_dev;
        struct in_ifaddr *ifa1;

        in_dev_for_each_ifa_rtnl_net(net, ifa1, in_dev) {
                if (ifa1->ifa_mask == ifa->ifa_mask &&
                    inet_ifa_match(ifa1->ifa_address, ifa) &&
                    ifa1->ifa_local == ifa->ifa_local)
                        return ifa1;
        }

        return NULL;
}

static int inet_rtm_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh,
                            struct netlink_ext_ack *extack)
{
        __u32 prefered_lft = INFINITY_LIFE_TIME;
        __u32 valid_lft = INFINITY_LIFE_TIME;
        struct net *net = sock_net(skb->sk);
        struct in_ifaddr *ifa_existing;
        struct nlattr *tb[IFA_MAX + 1];
        struct in_ifaddr *ifa;
        int ret;

        ret = inet_validate_rtm(nlh, tb, extack, &valid_lft, &prefered_lft);
        if (ret < 0)
                return ret;

        if (!nla_get_in_addr(tb[IFA_LOCAL]))
                return 0;

        rtnl_net_lock(net);

        ifa = inet_rtm_to_ifa(net, nlh, tb, extack);
        if (IS_ERR(ifa)) {
                ret = PTR_ERR(ifa);
                goto unlock;
        }

        ifa_existing = find_matching_ifa(net, ifa);
        if (!ifa_existing) {
                /* It would be best to check for !NLM_F_CREATE here but
                 * userspace already relies on not having to provide this.
                 */
                set_ifa_lifetime(ifa, valid_lft, prefered_lft);
                if (ifa->ifa_flags & IFA_F_MCAUTOJOIN) {
                        ret = ip_mc_autojoin_config(net, true, ifa);
                        if (ret < 0) {
                                NL_SET_ERR_MSG(extack, "ipv4: Multicast auto join failed");
                                inet_free_ifa(ifa);
                                goto unlock;
                        }
                }

                ret = __inet_insert_ifa(ifa, nlh, NETLINK_CB(skb).portid, extack);
        } else {
                u32 new_metric = ifa->ifa_rt_priority;
                u8 new_proto = ifa->ifa_proto;

                inet_free_ifa(ifa);

                if (nlh->nlmsg_flags & NLM_F_EXCL ||
                    !(nlh->nlmsg_flags & NLM_F_REPLACE)) {
                        NL_SET_ERR_MSG(extack, "ipv4: Address already assigned");
                        ret = -EEXIST;
                        goto unlock;
                }
                ifa = ifa_existing;

                if (ifa->ifa_rt_priority != new_metric) {
                        fib_modify_prefix_metric(ifa, new_metric);
                        ifa->ifa_rt_priority = new_metric;
                }

                ifa->ifa_proto = new_proto;

                set_ifa_lifetime(ifa, valid_lft, prefered_lft);
                cancel_delayed_work(&net->ipv4.addr_chk_work);
                queue_delayed_work(system_power_efficient_wq,
                                   &net->ipv4.addr_chk_work, 0);
                rtmsg_ifa(RTM_NEWADDR, ifa, nlh, NETLINK_CB(skb).portid);
        }

unlock:
        rtnl_net_unlock(net);

        return ret;
}

/*
 *        Determine a default network mask, based on the IP address.
 */

static int inet_abc_len(__be32 addr)
{
        int rc = -1;        /* Something else, probably a multicast. */

        if (ipv4_is_zeronet(addr) || ipv4_is_lbcast(addr))
                rc = 0;
        else {
                __u32 haddr = ntohl(addr);
                if (IN_CLASSA(haddr))
                        rc = 8;
                else if (IN_CLASSB(haddr))
                        rc = 16;
                else if (IN_CLASSC(haddr))
                        rc = 24;
                else if (IN_CLASSE(haddr))
                        rc = 32;
        }

        return rc;
}


int devinet_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr)
{
        struct sockaddr_in sin_orig;
        struct sockaddr_in *sin = (struct sockaddr_in *)&ifr->ifr_addr;
        struct in_ifaddr __rcu **ifap = NULL;
        struct in_device *in_dev;
        struct in_ifaddr *ifa = NULL;
        struct net_device *dev;
        char *colon;
        int ret = -EFAULT;
        int tryaddrmatch = 0;

        ifr->ifr_name[IFNAMSIZ - 1] = 0;

        /* save original address for comparison */
        memcpy(&sin_orig, sin, sizeof(*sin));

        colon = strchr(ifr->ifr_name, ':');
        if (colon)
                *colon = 0;

        dev_load(net, ifr->ifr_name);

        switch (cmd) {
        case SIOCGIFADDR:        /* Get interface address */
        case SIOCGIFBRDADDR:        /* Get the broadcast address */
        case SIOCGIFDSTADDR:        /* Get the destination address */
        case SIOCGIFNETMASK:        /* Get the netmask for the interface */
                /* Note that these ioctls will not sleep,
                   so that we do not impose a lock.
                   One day we will be forced to put shlock here (I mean SMP)
                 */
                tryaddrmatch = (sin_orig.sin_family == AF_INET);
                memset(sin, 0, sizeof(*sin));
                sin->sin_family = AF_INET;
                break;

        case SIOCSIFFLAGS:
                ret = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto out;
                break;
        case SIOCSIFADDR:        /* Set interface address (and family) */
        case SIOCSIFBRDADDR:        /* Set the broadcast address */
        case SIOCSIFDSTADDR:        /* Set the destination address */
        case SIOCSIFNETMASK:         /* Set the netmask for the interface */
                ret = -EPERM;
                if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
                        goto out;
                ret = -EINVAL;
                if (sin->sin_family != AF_INET)
                        goto out;
                break;
        default:
                ret = -EINVAL;
                goto out;
        }

        rtnl_net_lock(net);

        ret = -ENODEV;
        dev = __dev_get_by_name(net, ifr->ifr_name);
        if (!dev)
                goto done;

        if (colon)
                *colon = ':';

        in_dev = __in_dev_get_rtnl_net(dev);
        if (in_dev) {
                if (tryaddrmatch) {
                        /* Matthias Andree */
                        /* compare label and address (4.4BSD style) */
                        /* note: we only do this for a limited set of ioctls
                           and only if the original address family was AF_INET.
                           This is checked above. */

                        for (ifap = &in_dev->ifa_list;
                             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
                             ifap = &ifa->ifa_next) {
                                if (!strcmp(ifr->ifr_name, ifa->ifa_label) &&
                                    sin_orig.sin_addr.s_addr ==
                                                        ifa->ifa_local) {
                                        break; /* found */
                                }
                        }
                }
                /* we didn't get a match, maybe the application is
                   4.3BSD-style and passed in junk so we fall back to
                   comparing just the label */
                if (!ifa) {
                        for (ifap = &in_dev->ifa_list;
                             (ifa = rtnl_net_dereference(net, *ifap)) != NULL;
                             ifap = &ifa->ifa_next)
                                if (!strcmp(ifr->ifr_name, ifa->ifa_label))
                                        break;
                }
        }

        ret = -EADDRNOTAVAIL;
        if (!ifa && cmd != SIOCSIFADDR && cmd != SIOCSIFFLAGS)
                goto done;

        switch (cmd) {
        case SIOCGIFADDR:        /* Get interface address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_local;
                break;

        case SIOCGIFBRDADDR:        /* Get the broadcast address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_broadcast;
                break;

        case SIOCGIFDSTADDR:        /* Get the destination address */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_address;
                break;

        case SIOCGIFNETMASK:        /* Get the netmask for the interface */
                ret = 0;
                sin->sin_addr.s_addr = ifa->ifa_mask;
                break;

        case SIOCSIFFLAGS:
                if (colon) {
                        ret = -EADDRNOTAVAIL;
                        if (!ifa)
                                break;
                        ret = 0;
                        if (!(ifr->ifr_flags & IFF_UP))
                                inet_del_ifa(in_dev, ifap, 1);
                        break;
                }

                /* NETDEV_UP/DOWN/CHANGE could touch a peer dev */
                ASSERT_RTNL();
                ret = dev_change_flags(dev, ifr->ifr_flags, NULL);
                break;

        case SIOCSIFADDR:        /* Set interface address (and family) */
                ret = -EINVAL;
                if (inet_abc_len(sin->sin_addr.s_addr) < 0)
                        break;

                if (!ifa) {
                        ret = -ENOBUFS;
                        if (!in_dev)
                                break;
                        ifa = inet_alloc_ifa(in_dev);
                        if (!ifa)
                                break;

                        if (colon)
                                memcpy(ifa->ifa_label, ifr->ifr_name, IFNAMSIZ);
                        else
                                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                } else {
                        ret = 0;
                        if (ifa->ifa_local == sin->sin_addr.s_addr)
                                break;
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_broadcast = 0;
                        ifa->ifa_scope = 0;
                }

                ifa->ifa_address = ifa->ifa_local = sin->sin_addr.s_addr;

                if (!(dev->flags & IFF_POINTOPOINT)) {
                        ifa->ifa_prefixlen = inet_abc_len(ifa->ifa_address);
                        ifa->ifa_mask = inet_make_mask(ifa->ifa_prefixlen);
                        if ((dev->flags & IFF_BROADCAST) &&
                            ifa->ifa_prefixlen < 31)
                                ifa->ifa_broadcast = ifa->ifa_address |
                                                     ~ifa->ifa_mask;
                } else {
                        ifa->ifa_prefixlen = 32;
                        ifa->ifa_mask = inet_make_mask(32);
                }
                set_ifa_lifetime(ifa, INFINITY_LIFE_TIME, INFINITY_LIFE_TIME);
                ret = inet_set_ifa(dev, ifa);
                break;

        case SIOCSIFBRDADDR:        /* Set the broadcast address */
                ret = 0;
                if (ifa->ifa_broadcast != sin->sin_addr.s_addr) {
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_broadcast = sin->sin_addr.s_addr;
                        inet_insert_ifa(ifa);
                }
                break;

        case SIOCSIFDSTADDR:        /* Set the destination address */
                ret = 0;
                if (ifa->ifa_address == sin->sin_addr.s_addr)
                        break;
                ret = -EINVAL;
                if (inet_abc_len(sin->sin_addr.s_addr) < 0)
                        break;
                ret = 0;
                inet_del_ifa(in_dev, ifap, 0);
                ifa->ifa_address = sin->sin_addr.s_addr;
                inet_insert_ifa(ifa);
                break;

        case SIOCSIFNETMASK:         /* Set the netmask for the interface */

                /*
                 *        The mask we set must be legal.
                 */
                ret = -EINVAL;
                if (bad_mask(sin->sin_addr.s_addr, 0))
                        break;
                ret = 0;
                if (ifa->ifa_mask != sin->sin_addr.s_addr) {
                        __be32 old_mask = ifa->ifa_mask;
                        inet_del_ifa(in_dev, ifap, 0);
                        ifa->ifa_mask = sin->sin_addr.s_addr;
                        ifa->ifa_prefixlen = inet_mask_len(ifa->ifa_mask);

                        /* See if current broadcast address matches
                         * with current netmask, then recalculate
                         * the broadcast address. Otherwise it's a
                         * funny address, so don't touch it since
                         * the user seems to know what (s)he's doing...
                         */
                        if ((dev->flags & IFF_BROADCAST) &&
                            (ifa->ifa_prefixlen < 31) &&
                            (ifa->ifa_broadcast ==
                             (ifa->ifa_local|~old_mask))) {
                                ifa->ifa_broadcast = (ifa->ifa_local |
                                                      ~sin->sin_addr.s_addr);
                        }
                        inet_insert_ifa(ifa);
                }
                break;
        }
done:
        rtnl_net_unlock(net);
out:
        return ret;
}

int inet_gifconf(struct net_device *dev, char __user *buf, int len, int size)
{
        struct in_device *in_dev = __in_dev_get_rtnl_net(dev);
        const struct in_ifaddr *ifa;
        struct ifreq ifr;
        int done = 0;

        if (WARN_ON(size > sizeof(struct ifreq)))
                goto out;

        if (!in_dev)
                goto out;

        in_dev_for_each_ifa_rtnl_net(dev_net(dev), ifa, in_dev) {
                if (!buf) {
                        done += size;
                        continue;
                }
                if (len < size)
                        break;
                memset(&ifr, 0, sizeof(struct ifreq));
                strcpy(ifr.ifr_name, ifa->ifa_label);

                (*(struct sockaddr_in *)&ifr.ifr_addr).sin_family = AF_INET;
                (*(struct sockaddr_in *)&ifr.ifr_addr).sin_addr.s_addr =
                                                                ifa->ifa_local;

                if (copy_to_user(buf + done, &ifr, size)) {
                        done = -EFAULT;
                        break;
                }
                len  -= size;
                done += size;
        }
out:
        return done;
}

static __be32 in_dev_select_addr(const struct in_device *in_dev,
                                 int scope)
{
        const struct in_ifaddr *ifa;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (READ_ONCE(ifa->ifa_flags) & IFA_F_SECONDARY)
                        continue;
                if (ifa->ifa_scope != RT_SCOPE_LINK &&
                    ifa->ifa_scope <= scope)
                        return ifa->ifa_local;
        }

        return 0;
}

__be32 inet_select_addr(const struct net_device *dev, __be32 dst, int scope)
{
        const struct in_ifaddr *ifa;
        __be32 addr = 0;
        unsigned char localnet_scope = RT_SCOPE_HOST;
        struct in_device *in_dev;
        struct net *net;
        int master_idx;

        rcu_read_lock();
        net = dev_net_rcu(dev);
        in_dev = __in_dev_get_rcu(dev);
        if (!in_dev)
                goto no_in_dev;

        if (unlikely(IN_DEV_ROUTE_LOCALNET(in_dev)))
                localnet_scope = RT_SCOPE_LINK;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (READ_ONCE(ifa->ifa_flags) & IFA_F_SECONDARY)
                        continue;
                if (min(ifa->ifa_scope, localnet_scope) > scope)
                        continue;
                if (!dst || inet_ifa_match(dst, ifa)) {
                        addr = ifa->ifa_local;
                        break;
                }
                if (!addr)
                        addr = ifa->ifa_local;
        }

        if (addr)
                goto out_unlock;
no_in_dev:
        master_idx = l3mdev_master_ifindex_rcu(dev);

        /* For VRFs, the VRF device takes the place of the loopback device,
         * with addresses on it being preferred.  Note in such cases the
         * loopback device will be among the devices that fail the master_idx
         * equality check in the loop below.
         */
        if (master_idx &&
            (dev = dev_get_by_index_rcu(net, master_idx)) &&
            (in_dev = __in_dev_get_rcu(dev))) {
                addr = in_dev_select_addr(in_dev, scope);
                if (addr)
                        goto out_unlock;
        }

        /* Not loopback addresses on loopback should be preferred
           in this case. It is important that lo is the first interface
           in dev_base list.
         */
        for_each_netdev_rcu(net, dev) {
                if (l3mdev_master_ifindex_rcu(dev) != master_idx)
                        continue;

                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;

                addr = in_dev_select_addr(in_dev, scope);
                if (addr)
                        goto out_unlock;
        }
out_unlock:
        rcu_read_unlock();
        return addr;
}
EXPORT_SYMBOL(inet_select_addr);

static __be32 confirm_addr_indev(struct in_device *in_dev, __be32 dst,
                              __be32 local, int scope)
{
        unsigned char localnet_scope = RT_SCOPE_HOST;
        const struct in_ifaddr *ifa;
        __be32 addr = 0;
        int same = 0;

        if (unlikely(IN_DEV_ROUTE_LOCALNET(in_dev)))
                localnet_scope = RT_SCOPE_LINK;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                unsigned char min_scope = min(ifa->ifa_scope, localnet_scope);

                if (!addr &&
                    (local == ifa->ifa_local || !local) &&
                    min_scope <= scope) {
                        addr = ifa->ifa_local;
                        if (same)
                                break;
                }
                if (!same) {
                        same = (!local || inet_ifa_match(local, ifa)) &&
                                (!dst || inet_ifa_match(dst, ifa));
                        if (same && addr) {
                                if (local || !dst)
                                        break;
                                /* Is the selected addr into dst subnet? */
                                if (inet_ifa_match(addr, ifa))
                                        break;
                                /* No, then can we use new local src? */
                                if (min_scope <= scope) {
                                        addr = ifa->ifa_local;
                                        break;
                                }
                                /* search for large dst subnet for addr */
                                same = 0;
                        }
                }
        }

        return same ? addr : 0;
}

/*
 * Confirm that local IP address exists using wildcards:
 * - net: netns to check, cannot be NULL
 * - in_dev: only on this interface, NULL=any interface
 * - dst: only in the same subnet as dst, 0=any dst
 * - local: address, 0=autoselect the local address
 * - scope: maximum allowed scope value for the local address
 */
__be32 inet_confirm_addr(struct net *net, struct in_device *in_dev,
                         __be32 dst, __be32 local, int scope)
{
        __be32 addr = 0;
        struct net_device *dev;

        if (in_dev)
                return confirm_addr_indev(in_dev, dst, local, scope);

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                in_dev = __in_dev_get_rcu(dev);
                if (in_dev) {
                        addr = confirm_addr_indev(in_dev, dst, local, scope);
                        if (addr)
                                break;
                }
        }
        rcu_read_unlock();

        return addr;
}
EXPORT_SYMBOL(inet_confirm_addr);

/*
 *        Device notifier
 */

int register_inetaddr_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&inetaddr_chain, nb);
}
EXPORT_SYMBOL(register_inetaddr_notifier);

int unregister_inetaddr_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&inetaddr_chain, nb);
}
EXPORT_SYMBOL(unregister_inetaddr_notifier);

int register_inetaddr_validator_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&inetaddr_validator_chain, nb);
}
EXPORT_SYMBOL(register_inetaddr_validator_notifier);

int unregister_inetaddr_validator_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&inetaddr_validator_chain,
            nb);
}
EXPORT_SYMBOL(unregister_inetaddr_validator_notifier);

/* Rename ifa_labels for a device name change. Make some effort to preserve
 * existing alias numbering and to create unique labels if possible.
*/
static void inetdev_changename(struct net_device *dev, struct in_device *in_dev)
{
        struct in_ifaddr *ifa;
        int named = 0;

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                char old[IFNAMSIZ], *dot;

                memcpy(old, ifa->ifa_label, IFNAMSIZ);
                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                if (named++ == 0)
                        goto skip;
                dot = strchr(old, ':');
                if (!dot) {
                        sprintf(old, ":%d", named);
                        dot = old;
                }
                if (strlen(dot) + strlen(dev->name) < IFNAMSIZ)
                        strcat(ifa->ifa_label, dot);
                else
                        strcpy(ifa->ifa_label + (IFNAMSIZ - strlen(dot) - 1), dot);
skip:
                rtmsg_ifa(RTM_NEWADDR, ifa, NULL, 0);
        }
}

static void inetdev_send_gratuitous_arp(struct net_device *dev,
                                        struct in_device *in_dev)

{
        const struct in_ifaddr *ifa;

        in_dev_for_each_ifa_rtnl(ifa, in_dev) {
                arp_send(ARPOP_REQUEST, ETH_P_ARP,
                         ifa->ifa_local, dev,
                         ifa->ifa_local, NULL,
                         dev->dev_addr, NULL);
        }
}

/* Called only under RTNL semaphore */

static int inetdev_event(struct notifier_block *this, unsigned long event,
                         void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct in_device *in_dev = __in_dev_get_rtnl(dev);

        ASSERT_RTNL();

        if (!in_dev) {
                if (event == NETDEV_REGISTER) {
                        in_dev = inetdev_init(dev);
                        if (IS_ERR(in_dev))
                                return notifier_from_errno(PTR_ERR(in_dev));
                        if (dev->flags & IFF_LOOPBACK) {
                                IN_DEV_CONF_SET(in_dev, NOXFRM, 1);
                                IN_DEV_CONF_SET(in_dev, NOPOLICY, 1);
                        }
                } else if (event == NETDEV_CHANGEMTU) {
                        /* Re-enabling IP */
                        if (inetdev_valid_mtu(dev->mtu))
                                in_dev = inetdev_init(dev);
                }
                goto out;
        }

        switch (event) {
        case NETDEV_REGISTER:
                pr_debug("%s: bug\n", __func__);
                RCU_INIT_POINTER(dev->ip_ptr, NULL);
                break;
        case NETDEV_UP:
                if (!inetdev_valid_mtu(dev->mtu))
                        break;
                if (dev->flags & IFF_LOOPBACK) {
                        struct in_ifaddr *ifa = inet_alloc_ifa(in_dev);

                        if (ifa) {
                                ifa->ifa_local =
                                  ifa->ifa_address = htonl(INADDR_LOOPBACK);
                                ifa->ifa_prefixlen = 8;
                                ifa->ifa_mask = inet_make_mask(8);
                                ifa->ifa_scope = RT_SCOPE_HOST;
                                memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
                                set_ifa_lifetime(ifa, INFINITY_LIFE_TIME,
                                                 INFINITY_LIFE_TIME);
                                ipv4_devconf_setall(in_dev);
                                neigh_parms_data_state_setall(in_dev->arp_parms);
                                inet_insert_ifa(ifa);
                        }
                }
                ip_mc_up(in_dev);
                fallthrough;
        case NETDEV_CHANGEADDR:
                if (!IN_DEV_ARP_NOTIFY(in_dev))
                        break;
                fallthrough;
        case NETDEV_NOTIFY_PEERS:
                /* Send gratuitous ARP to notify of link change */
                inetdev_send_gratuitous_arp(dev, in_dev);
                break;
        case NETDEV_DOWN:
                ip_mc_down(in_dev);
                break;
        case NETDEV_PRE_TYPE_CHANGE:
                ip_mc_unmap(in_dev);
                break;
        case NETDEV_POST_TYPE_CHANGE:
                ip_mc_remap(in_dev);
                break;
        case NETDEV_CHANGEMTU:
                if (inetdev_valid_mtu(dev->mtu))
                        break;
                /* disable IP when MTU is not enough */
                fallthrough;
        case NETDEV_UNREGISTER:
                inetdev_destroy(in_dev);
                break;
        case NETDEV_CHANGENAME:
                /* Do not notify about label change, this event is
                 * not interesting to applications using netlink.
                 */
                inetdev_changename(dev, in_dev);

                devinet_sysctl_unregister(in_dev);
                devinet_sysctl_register(in_dev);
                break;
        }
out:
        return NOTIFY_DONE;
}

static struct notifier_block ip_netdev_notifier = {
        .notifier_call = inetdev_event,
};

static size_t inet_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ifaddrmsg))
               + nla_total_size(4) /* IFA_ADDRESS */
               + nla_total_size(4) /* IFA_LOCAL */
               + nla_total_size(4) /* IFA_BROADCAST */
               + nla_total_size(IFNAMSIZ) /* IFA_LABEL */
               + nla_total_size(4)  /* IFA_FLAGS */
               + nla_total_size(1)  /* IFA_PROTO */
               + nla_total_size(4)  /* IFA_RT_PRIORITY */
               + nla_total_size(sizeof(struct ifa_cacheinfo)); /* IFA_CACHEINFO */
}

static inline u32 cstamp_delta(unsigned long cstamp)
{
        return (cstamp - INITIAL_JIFFIES) * 100UL / HZ;
}

static int put_cacheinfo(struct sk_buff *skb, unsigned long cstamp,
                         unsigned long tstamp, u32 preferred, u32 valid)
{
        struct ifa_cacheinfo ci;

        ci.cstamp = cstamp_delta(cstamp);
        ci.tstamp = cstamp_delta(tstamp);
        ci.ifa_prefered = preferred;
        ci.ifa_valid = valid;

        return nla_put(skb, IFA_CACHEINFO, sizeof(ci), &ci);
}

static int inet_fill_ifaddr(struct sk_buff *skb, const struct in_ifaddr *ifa,
                            struct inet_fill_args *args)
{
        struct ifaddrmsg *ifm;
        struct nlmsghdr  *nlh;
        unsigned long tstamp;
        u32 preferred, valid;
        u32 flags;

        nlh = nlmsg_put(skb, args->portid, args->seq, args->event, sizeof(*ifm),
                        args->flags);
        if (!nlh)
                return -EMSGSIZE;

        ifm = nlmsg_data(nlh);
        ifm->ifa_family = AF_INET;
        ifm->ifa_prefixlen = ifa->ifa_prefixlen;

        flags = READ_ONCE(ifa->ifa_flags);
        /* Warning : ifm->ifa_flags is an __u8, it holds only 8 bits.
         * The 32bit value is given in IFA_FLAGS attribute.
         */
        ifm->ifa_flags = (__u8)flags;

        ifm->ifa_scope = ifa->ifa_scope;
        ifm->ifa_index = ifa->ifa_dev->dev->ifindex;

        if (args->netnsid >= 0 &&
            nla_put_s32(skb, IFA_TARGET_NETNSID, args->netnsid))
                goto nla_put_failure;

        tstamp = READ_ONCE(ifa->ifa_tstamp);
        if (!(flags & IFA_F_PERMANENT)) {
                preferred = READ_ONCE(ifa->ifa_preferred_lft);
                valid = READ_ONCE(ifa->ifa_valid_lft);
                if (preferred != INFINITY_LIFE_TIME) {
                        long tval = (jiffies - tstamp) / HZ;

                        if (preferred > tval)
                                preferred -= tval;
                        else
                                preferred = 0;
                        if (valid != INFINITY_LIFE_TIME) {
                                if (valid > tval)
                                        valid -= tval;
                                else
                                        valid = 0;
                        }
                }
        } else {
                preferred = INFINITY_LIFE_TIME;
                valid = INFINITY_LIFE_TIME;
        }
        if ((ifa->ifa_address &&
             nla_put_in_addr(skb, IFA_ADDRESS, ifa->ifa_address)) ||
            (ifa->ifa_local &&
             nla_put_in_addr(skb, IFA_LOCAL, ifa->ifa_local)) ||
            (ifa->ifa_broadcast &&
             nla_put_in_addr(skb, IFA_BROADCAST, ifa->ifa_broadcast)) ||
            (ifa->ifa_label[0] &&
             nla_put_string(skb, IFA_LABEL, ifa->ifa_label)) ||
            (ifa->ifa_proto &&
             nla_put_u8(skb, IFA_PROTO, ifa->ifa_proto)) ||
            nla_put_u32(skb, IFA_FLAGS, flags) ||
            (ifa->ifa_rt_priority &&
             nla_put_u32(skb, IFA_RT_PRIORITY, ifa->ifa_rt_priority)) ||
            put_cacheinfo(skb, READ_ONCE(ifa->ifa_cstamp), tstamp,
                          preferred, valid))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int inet_valid_dump_ifaddr_req(const struct nlmsghdr *nlh,
                                      struct inet_fill_args *fillargs,
                                      struct net **tgt_net, struct sock *sk,
                                      struct netlink_callback *cb)
{
        struct netlink_ext_ack *extack = cb->extack;
        struct nlattr *tb[IFA_MAX+1];
        struct ifaddrmsg *ifm;
        int err, i;

        ifm = nlmsg_payload(nlh, sizeof(*ifm));
        if (!ifm) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid header for address dump request");
                return -EINVAL;
        }

        if (ifm->ifa_prefixlen || ifm->ifa_flags || ifm->ifa_scope) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for address dump request");
                return -EINVAL;
        }

        fillargs->ifindex = ifm->ifa_index;
        if (fillargs->ifindex) {
                cb->answer_flags |= NLM_F_DUMP_FILTERED;
                fillargs->flags |= NLM_F_DUMP_FILTERED;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(*ifm), tb, IFA_MAX,
                                            ifa_ipv4_policy, extack);
        if (err < 0)
                return err;

        for (i = 0; i <= IFA_MAX; ++i) {
                if (!tb[i])
                        continue;

                if (i == IFA_TARGET_NETNSID) {
                        struct net *net;

                        fillargs->netnsid = nla_get_s32(tb[i]);

                        net = rtnl_get_net_ns_capable(sk, fillargs->netnsid);
                        if (IS_ERR(net)) {
                                fillargs->netnsid = -1;
                                NL_SET_ERR_MSG(extack, "ipv4: Invalid target network namespace id");
                                return PTR_ERR(net);
                        }
                        *tgt_net = net;
                } else {
                        NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in dump request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int in_dev_dump_ifmcaddr(struct in_device *in_dev, struct sk_buff *skb,
                                struct netlink_callback *cb, int *s_ip_idx,
                                struct inet_fill_args *fillargs)
{
        struct ip_mc_list *im;
        int ip_idx = 0;
        int err;

        for (im = rcu_dereference(in_dev->mc_list);
             im;
             im = rcu_dereference(im->next_rcu)) {
                if (ip_idx < *s_ip_idx) {
                        ip_idx++;
                        continue;
                }
                err = inet_fill_ifmcaddr(skb, in_dev->dev, im, fillargs);
                if (err < 0)
                        goto done;

                nl_dump_check_consistent(cb, nlmsg_hdr(skb));
                ip_idx++;
        }
        err = 0;
        ip_idx = 0;
done:
        *s_ip_idx = ip_idx;
        return err;
}

static int in_dev_dump_ifaddr(struct in_device *in_dev, struct sk_buff *skb,
                              struct netlink_callback *cb, int *s_ip_idx,
                              struct inet_fill_args *fillargs)
{
        struct in_ifaddr *ifa;
        int ip_idx = 0;
        int err;

        in_dev_for_each_ifa_rcu(ifa, in_dev) {
                if (ip_idx < *s_ip_idx) {
                        ip_idx++;
                        continue;
                }
                err = inet_fill_ifaddr(skb, ifa, fillargs);
                if (err < 0)
                        goto done;

                nl_dump_check_consistent(cb, nlmsg_hdr(skb));
                ip_idx++;
        }
        err = 0;
        ip_idx = 0;
done:
        *s_ip_idx = ip_idx;

        return err;
}

static int in_dev_dump_addr(struct in_device *in_dev, struct sk_buff *skb,
                            struct netlink_callback *cb, int *s_ip_idx,
                            struct inet_fill_args *fillargs)
{
        switch (fillargs->event) {
        case RTM_NEWADDR:
                return in_dev_dump_ifaddr(in_dev, skb, cb, s_ip_idx, fillargs);
        case RTM_GETMULTICAST:
                return in_dev_dump_ifmcaddr(in_dev, skb, cb, s_ip_idx,
                                            fillargs);
        default:
                return -EINVAL;
        }
}

/* Combine dev_addr_genid and dev_base_seq to detect changes.
 */
static u32 inet_base_seq(const struct net *net)
{
        u32 res = atomic_read(&net->ipv4.dev_addr_genid) +
                  READ_ONCE(net->dev_base_seq);

        /* Must not return 0 (see nl_dump_check_consistent()).
         * Chose a value far away from 0.
         */
        if (!res)
                res = 0x80000000;
        return res;
}

static int inet_dump_addr(struct sk_buff *skb, struct netlink_callback *cb,
                          int event)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct inet_fill_args fillargs = {
                .portid = NETLINK_CB(cb->skb).portid,
                .seq = nlh->nlmsg_seq,
                .event = event,
                .flags = NLM_F_MULTI,
                .netnsid = -1,
        };
        struct net *net = sock_net(skb->sk);
        struct net *tgt_net = net;
        struct {
                unsigned long ifindex;
                int ip_idx;
        } *ctx = (void *)cb->ctx;
        struct in_device *in_dev;
        struct net_device *dev;
        int err = 0;

        rcu_read_lock();
        if (cb->strict_check) {
                err = inet_valid_dump_ifaddr_req(nlh, &fillargs, &tgt_net,
                                                 skb->sk, cb);
                if (err < 0)
                        goto done;

                if (fillargs.ifindex) {
                        dev = dev_get_by_index_rcu(tgt_net, fillargs.ifindex);
                        if (!dev) {
                                err = -ENODEV;
                                goto done;
                        }
                        in_dev = __in_dev_get_rcu(dev);
                        if (!in_dev)
                                goto done;
                        err = in_dev_dump_addr(in_dev, skb, cb, &ctx->ip_idx,
                                               &fillargs);
                        goto done;
                }
        }

        cb->seq = inet_base_seq(tgt_net);

        for_each_netdev_dump(tgt_net, dev, ctx->ifindex) {
                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;
                err = in_dev_dump_addr(in_dev, skb, cb, &ctx->ip_idx,
                                       &fillargs);
                if (err < 0)
                        goto done;
        }
done:
        if (fillargs.netnsid >= 0)
                put_net(tgt_net);
        rcu_read_unlock();
        return err;
}

static int inet_dump_ifaddr(struct sk_buff *skb, struct netlink_callback *cb)
{
        return inet_dump_addr(skb, cb, RTM_NEWADDR);
}

static int inet_dump_ifmcaddr(struct sk_buff *skb, struct netlink_callback *cb)
{
        return inet_dump_addr(skb, cb, RTM_GETMULTICAST);
}

static void rtmsg_ifa(int event, struct in_ifaddr *ifa, struct nlmsghdr *nlh,
                      u32 portid)
{
        struct inet_fill_args fillargs = {
                .portid = portid,
                .seq = nlh ? nlh->nlmsg_seq : 0,
                .event = event,
                .flags = 0,
                .netnsid = -1,
        };
        struct sk_buff *skb;
        int err = -ENOBUFS;
        struct net *net;

        net = dev_net(ifa->ifa_dev->dev);
        skb = nlmsg_new(inet_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_fill_ifaddr(skb, ifa, &fillargs);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, portid, RTNLGRP_IPV4_IFADDR, nlh, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV4_IFADDR, err);
}

static size_t inet_get_link_af_size(const struct net_device *dev,
                                    u32 ext_filter_mask)
{
        struct in_device *in_dev = rcu_dereference_rtnl(dev->ip_ptr);

        if (!in_dev)
                return 0;

        return nla_total_size(IPV4_DEVCONF_MAX * 4); /* IFLA_INET_CONF */
}

static int inet_fill_link_af(struct sk_buff *skb, const struct net_device *dev,
                             u32 ext_filter_mask)
{
        struct in_device *in_dev = rcu_dereference_rtnl(dev->ip_ptr);
        struct nlattr *nla;
        int i;

        if (!in_dev)
                return -ENODATA;

        nla = nla_reserve(skb, IFLA_INET_CONF, IPV4_DEVCONF_MAX * 4);
        if (!nla)
                return -EMSGSIZE;

        for (i = 0; i < IPV4_DEVCONF_MAX; i++)
                ((u32 *) nla_data(nla))[i] = READ_ONCE(in_dev->cnf.data[i]);

        return 0;
}

static const struct nla_policy inet_af_policy[IFLA_INET_MAX+1] = {
        [IFLA_INET_CONF]        = { .type = NLA_NESTED },
};

static int inet_validate_link_af(const struct net_device *dev,
                                 const struct nlattr *nla,
                                 struct netlink_ext_ack *extack)
{
        struct nlattr *a, *tb[IFLA_INET_MAX+1];
        int err, rem;

        if (dev && !__in_dev_get_rtnl(dev))
                return -EAFNOSUPPORT;

        err = nla_parse_nested_deprecated(tb, IFLA_INET_MAX, nla,
                                          inet_af_policy, extack);
        if (err < 0)
                return err;

        if (tb[IFLA_INET_CONF]) {
                nla_for_each_nested(a, tb[IFLA_INET_CONF], rem) {
                        int cfgid = nla_type(a);

                        if (nla_len(a) < 4)
                                return -EINVAL;

                        if (cfgid <= 0 || cfgid > IPV4_DEVCONF_MAX)
                                return -EINVAL;
                }
        }

        return 0;
}

static int inet_set_link_af(struct net_device *dev, const struct nlattr *nla,
                            struct netlink_ext_ack *extack)
{
        struct in_device *in_dev = __in_dev_get_rtnl(dev);
        struct nlattr *a, *tb[IFLA_INET_MAX+1];
        int rem;

        if (!in_dev)
                return -EAFNOSUPPORT;

        if (nla_parse_nested_deprecated(tb, IFLA_INET_MAX, nla, NULL, NULL) < 0)
                return -EINVAL;

        if (tb[IFLA_INET_CONF]) {
                nla_for_each_nested(a, tb[IFLA_INET_CONF], rem)
                        ipv4_devconf_set(in_dev, nla_type(a), nla_get_u32(a));
        }

        return 0;
}

static int inet_netconf_msgsize_devconf(int type)
{
        int size = NLMSG_ALIGN(sizeof(struct netconfmsg))
                   + nla_total_size(4);        /* NETCONFA_IFINDEX */
        bool all = false;

        if (type == NETCONFA_ALL)
                all = true;

        if (all || type == NETCONFA_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_RP_FILTER)
                size += nla_total_size(4);
        if (all || type == NETCONFA_MC_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_BC_FORWARDING)
                size += nla_total_size(4);
        if (all || type == NETCONFA_PROXY_NEIGH)
                size += nla_total_size(4);
        if (all || type == NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN)
                size += nla_total_size(4);

        return size;
}

static int inet_netconf_fill_devconf(struct sk_buff *skb, int ifindex,
                                     const struct ipv4_devconf *devconf,
                                     u32 portid, u32 seq, int event,
                                     unsigned int flags, int type)
{
        struct nlmsghdr  *nlh;
        struct netconfmsg *ncm;
        bool all = false;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(struct netconfmsg),
                        flags);
        if (!nlh)
                return -EMSGSIZE;

        if (type == NETCONFA_ALL)
                all = true;

        ncm = nlmsg_data(nlh);
        ncm->ncm_family = AF_INET;

        if (nla_put_s32(skb, NETCONFA_IFINDEX, ifindex) < 0)
                goto nla_put_failure;

        if (!devconf)
                goto out;

        if ((all || type == NETCONFA_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_RP_FILTER) &&
            nla_put_s32(skb, NETCONFA_RP_FILTER,
                        IPV4_DEVCONF_RO(*devconf, RP_FILTER)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_MC_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_MC_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, MC_FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_BC_FORWARDING) &&
            nla_put_s32(skb, NETCONFA_BC_FORWARDING,
                        IPV4_DEVCONF_RO(*devconf, BC_FORWARDING)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_PROXY_NEIGH) &&
            nla_put_s32(skb, NETCONFA_PROXY_NEIGH,
                        IPV4_DEVCONF_RO(*devconf, PROXY_ARP)) < 0)
                goto nla_put_failure;
        if ((all || type == NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN) &&
            nla_put_s32(skb, NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN,
                        IPV4_DEVCONF_RO(*devconf,
                                        IGNORE_ROUTES_WITH_LINKDOWN)) < 0)
                goto nla_put_failure;

out:
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

void inet_netconf_notify_devconf(struct net *net, int event, int type,
                                 int ifindex, struct ipv4_devconf *devconf)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(inet_netconf_msgsize_devconf(type), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_netconf_fill_devconf(skb, ifindex, devconf, 0, 0,
                                        event, 0, type);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, 0, RTNLGRP_IPV4_NETCONF, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV4_NETCONF, err);
}

static const struct nla_policy devconf_ipv4_policy[NETCONFA_MAX+1] = {
        [NETCONFA_IFINDEX]        = { .len = sizeof(int) },
        [NETCONFA_FORWARDING]        = { .len = sizeof(int) },
        [NETCONFA_RP_FILTER]        = { .len = sizeof(int) },
        [NETCONFA_PROXY_NEIGH]        = { .len = sizeof(int) },
        [NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN]        = { .len = sizeof(int) },
};

static int inet_netconf_valid_get_req(struct sk_buff *skb,
                                      const struct nlmsghdr *nlh,
                                      struct nlattr **tb,
                                      struct netlink_ext_ack *extack)
{
        int i, err;

        if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(struct netconfmsg))) {
                NL_SET_ERR_MSG(extack, "ipv4: Invalid header for netconf get request");
                return -EINVAL;
        }

        if (!netlink_strict_get_check(skb))
                return nlmsg_parse_deprecated(nlh, sizeof(struct netconfmsg),
                                              tb, NETCONFA_MAX,
                                              devconf_ipv4_policy, extack);

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct netconfmsg),
                                            tb, NETCONFA_MAX,
                                            devconf_ipv4_policy, extack);
        if (err)
                return err;

        for (i = 0; i <= NETCONFA_MAX; i++) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NETCONFA_IFINDEX:
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in netconf get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static int inet_netconf_get_devconf(struct sk_buff *in_skb,
                                    struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nlattr *tb[NETCONFA_MAX + 1];
        const struct ipv4_devconf *devconf;
        struct in_device *in_dev = NULL;
        struct net_device *dev = NULL;
        struct sk_buff *skb;
        int ifindex;
        int err;

        err = inet_netconf_valid_get_req(in_skb, nlh, tb, extack);
        if (err)
                return err;

        if (!tb[NETCONFA_IFINDEX])
                return -EINVAL;

        ifindex = nla_get_s32(tb[NETCONFA_IFINDEX]);
        switch (ifindex) {
        case NETCONFA_IFINDEX_ALL:
                devconf = net->ipv4.devconf_all;
                break;
        case NETCONFA_IFINDEX_DEFAULT:
                devconf = net->ipv4.devconf_dflt;
                break;
        default:
                err = -ENODEV;
                dev = dev_get_by_index(net, ifindex);
                if (dev)
                        in_dev = in_dev_get(dev);
                if (!in_dev)
                        goto errout;
                devconf = &in_dev->cnf;
                break;
        }

        err = -ENOBUFS;
        skb = nlmsg_new(inet_netconf_msgsize_devconf(NETCONFA_ALL), GFP_KERNEL);
        if (!skb)
                goto errout;

        err = inet_netconf_fill_devconf(skb, ifindex, devconf,
                                        NETLINK_CB(in_skb).portid,
                                        nlh->nlmsg_seq, RTM_NEWNETCONF, 0,
                                        NETCONFA_ALL);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in inet_netconf_msgsize_devconf() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
errout:
        if (in_dev)
                in_dev_put(in_dev);
        dev_put(dev);
        return err;
}

static int inet_netconf_dump_devconf(struct sk_buff *skb,
                                     struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        struct {
                unsigned long ifindex;
                unsigned int all_default;
        } *ctx = (void *)cb->ctx;
        const struct in_device *in_dev;
        struct net_device *dev;
        int err = 0;

        if (cb->strict_check) {
                struct netlink_ext_ack *extack = cb->extack;
                struct netconfmsg *ncm;

                if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ncm))) {
                        NL_SET_ERR_MSG(extack, "ipv4: Invalid header for netconf dump request");
                        return -EINVAL;
                }

                if (nlmsg_attrlen(nlh, sizeof(*ncm))) {
                        NL_SET_ERR_MSG(extack, "ipv4: Invalid data after header in netconf dump request");
                        return -EINVAL;
                }
        }

        rcu_read_lock();
        for_each_netdev_dump(net, dev, ctx->ifindex) {
                in_dev = __in_dev_get_rcu(dev);
                if (!in_dev)
                        continue;
                err = inet_netconf_fill_devconf(skb, dev->ifindex,
                                                &in_dev->cnf,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
        }
        if (ctx->all_default == 0) {
                err = inet_netconf_fill_devconf(skb, NETCONFA_IFINDEX_ALL,
                                                net->ipv4.devconf_all,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
                ctx->all_default++;
        }
        if (ctx->all_default == 1) {
                err = inet_netconf_fill_devconf(skb, NETCONFA_IFINDEX_DEFAULT,
                                                net->ipv4.devconf_dflt,
                                                NETLINK_CB(cb->skb).portid,
                                                nlh->nlmsg_seq,
                                                RTM_NEWNETCONF, NLM_F_MULTI,
                                                NETCONFA_ALL);
                if (err < 0)
                        goto done;
                ctx->all_default++;
        }
done:
        rcu_read_unlock();
        return err;
}

#ifdef CONFIG_SYSCTL

static void devinet_copy_dflt_conf(struct net *net, int i)
{
        struct net_device *dev;

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                struct in_device *in_dev;

                in_dev = __in_dev_get_rcu(dev);
                if (in_dev && !test_bit(i, in_dev->cnf.state))
                        in_dev->cnf.data[i] = net->ipv4.devconf_dflt->data[i];
        }
        rcu_read_unlock();
}

/* called with RTNL locked */
static void inet_forward_change(struct net *net)
{
        struct net_device *dev;
        int on = IPV4_DEVCONF_ALL(net, FORWARDING);

        IPV4_DEVCONF_ALL(net, ACCEPT_REDIRECTS) = !on;
        IPV4_DEVCONF_DFLT(net, FORWARDING) = on;
        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                    NETCONFA_FORWARDING,
                                    NETCONFA_IFINDEX_ALL,
                                    net->ipv4.devconf_all);
        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                    NETCONFA_FORWARDING,
                                    NETCONFA_IFINDEX_DEFAULT,
                                    net->ipv4.devconf_dflt);

        for_each_netdev(net, dev) {
                struct in_device *in_dev;

                if (on)
                        dev_disable_lro(dev);

                in_dev = __in_dev_get_rtnl_net(dev);
                if (in_dev) {
                        IN_DEV_CONF_SET(in_dev, FORWARDING, on);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_FORWARDING,
                                                    dev->ifindex, &in_dev->cnf);
                }
        }
}

static int devinet_conf_ifindex(struct net *net, struct ipv4_devconf *cnf)
{
        if (cnf == net->ipv4.devconf_dflt)
                return NETCONFA_IFINDEX_DEFAULT;
        else if (cnf == net->ipv4.devconf_all)
                return NETCONFA_IFINDEX_ALL;
        else {
                struct in_device *idev
                        = container_of(cnf, struct in_device, cnf);
                return idev->dev->ifindex;
        }
}

static int devinet_conf_proc(const struct ctl_table *ctl, int write,
                             void *buffer, size_t *lenp, loff_t *ppos)
{
        int old_value = *(int *)ctl->data;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
        int new_value = *(int *)ctl->data;

        if (write) {
                struct ipv4_devconf *cnf = ctl->extra1;
                struct net *net = ctl->extra2;
                int i = (int *)ctl->data - cnf->data;
                int ifindex;

                set_bit(i, cnf->state);

                if (cnf == net->ipv4.devconf_dflt)
                        devinet_copy_dflt_conf(net, i);
                if (i == IPV4_DEVCONF_ACCEPT_LOCAL - 1 ||
                    i == IPV4_DEVCONF_ROUTE_LOCALNET - 1)
                        if ((new_value == 0) && (old_value != 0))
                                rt_cache_flush(net);

                if (i == IPV4_DEVCONF_BC_FORWARDING - 1 &&
                    new_value != old_value)
                        rt_cache_flush(net);

                if (i == IPV4_DEVCONF_RP_FILTER - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_RP_FILTER,
                                                    ifindex, cnf);
                }
                if (i == IPV4_DEVCONF_PROXY_ARP - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_PROXY_NEIGH,
                                                    ifindex, cnf);
                }
                if (i == IPV4_DEVCONF_IGNORE_ROUTES_WITH_LINKDOWN - 1 &&
                    new_value != old_value) {
                        ifindex = devinet_conf_ifindex(net, cnf);
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_IGNORE_ROUTES_WITH_LINKDOWN,
                                                    ifindex, cnf);
                }
        }

        return ret;
}

static int devinet_sysctl_forward(const struct ctl_table *ctl, int write,
                                  void *buffer, size_t *lenp, loff_t *ppos)
{
        int *valp = ctl->data;
        int val = *valp;
        loff_t pos = *ppos;
        struct net *net = ctl->extra2;
        int ret;

        if (write && !ns_capable(net->user_ns, CAP_NET_ADMIN))
                return -EPERM;

        ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        if (write && *valp != val) {
                if (valp != &IPV4_DEVCONF_DFLT(net, FORWARDING)) {
                        if (!rtnl_net_trylock(net)) {
                                /* Restore the original values before restarting */
                                *valp = val;
                                *ppos = pos;
                                return restart_syscall();
                        }
                        if (valp == &IPV4_DEVCONF_ALL(net, FORWARDING)) {
                                inet_forward_change(net);
                        } else {
                                struct ipv4_devconf *cnf = ctl->extra1;
                                struct in_device *idev =
                                        container_of(cnf, struct in_device, cnf);
                                if (*valp)
                                        dev_disable_lro(idev->dev);
                                inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                            NETCONFA_FORWARDING,
                                                            idev->dev->ifindex,
                                                            cnf);
                        }
                        rtnl_net_unlock(net);
                        rt_cache_flush(net);
                } else
                        inet_netconf_notify_devconf(net, RTM_NEWNETCONF,
                                                    NETCONFA_FORWARDING,
                                                    NETCONFA_IFINDEX_DEFAULT,
                                                    net->ipv4.devconf_dflt);
        }

        return ret;
}

static int ipv4_doint_and_flush(const struct ctl_table *ctl, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        int *valp = ctl->data;
        int val = *valp;
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
        struct net *net = ctl->extra2;

        if (write && *valp != val)
                rt_cache_flush(net);

        return ret;
}

#define DEVINET_SYSCTL_ENTRY(attr, name, mval, proc) \
        { \
                .procname        = name, \
                .data                = ipv4_devconf.data + \
                                  IPV4_DEVCONF_ ## attr - 1, \
                .maxlen                = sizeof(int), \
                .mode                = mval, \
                .proc_handler        = proc, \
                .extra1                = &ipv4_devconf, \
        }

#define DEVINET_SYSCTL_RW_ENTRY(attr, name) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0644, devinet_conf_proc)

#define DEVINET_SYSCTL_RO_ENTRY(attr, name) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0444, devinet_conf_proc)

#define DEVINET_SYSCTL_COMPLEX_ENTRY(attr, name, proc) \
        DEVINET_SYSCTL_ENTRY(attr, name, 0644, proc)

#define DEVINET_SYSCTL_FLUSHING_ENTRY(attr, name) \
        DEVINET_SYSCTL_COMPLEX_ENTRY(attr, name, ipv4_doint_and_flush)

static struct devinet_sysctl_table {
        struct ctl_table_header *sysctl_header;
        struct ctl_table devinet_vars[IPV4_DEVCONF_MAX];
} devinet_sysctl = {
        .devinet_vars = {
                DEVINET_SYSCTL_COMPLEX_ENTRY(FORWARDING, "forwarding",
                                             devinet_sysctl_forward),
                DEVINET_SYSCTL_RO_ENTRY(MC_FORWARDING, "mc_forwarding"),
                DEVINET_SYSCTL_RW_ENTRY(BC_FORWARDING, "bc_forwarding"),

                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_REDIRECTS, "accept_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(SECURE_REDIRECTS, "secure_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(SHARED_MEDIA, "shared_media"),
                DEVINET_SYSCTL_RW_ENTRY(RP_FILTER, "rp_filter"),
                DEVINET_SYSCTL_RW_ENTRY(SEND_REDIRECTS, "send_redirects"),
                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_SOURCE_ROUTE,
                                        "accept_source_route"),
                DEVINET_SYSCTL_RW_ENTRY(ACCEPT_LOCAL, "accept_local"),
                DEVINET_SYSCTL_RW_ENTRY(SRC_VMARK, "src_valid_mark"),
                DEVINET_SYSCTL_RW_ENTRY(PROXY_ARP, "proxy_arp"),
                DEVINET_SYSCTL_RW_ENTRY(MEDIUM_ID, "medium_id"),
                DEVINET_SYSCTL_RW_ENTRY(BOOTP_RELAY, "bootp_relay"),
                DEVINET_SYSCTL_RW_ENTRY(LOG_MARTIANS, "log_martians"),
                DEVINET_SYSCTL_RW_ENTRY(TAG, "tag"),
                DEVINET_SYSCTL_RW_ENTRY(ARPFILTER, "arp_filter"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_ANNOUNCE, "arp_announce"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_IGNORE, "arp_ignore"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_ACCEPT, "arp_accept"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_NOTIFY, "arp_notify"),
                DEVINET_SYSCTL_RW_ENTRY(ARP_EVICT_NOCARRIER,
                                        "arp_evict_nocarrier"),
                DEVINET_SYSCTL_RW_ENTRY(PROXY_ARP_PVLAN, "proxy_arp_pvlan"),
                DEVINET_SYSCTL_RW_ENTRY(FORCE_IGMP_VERSION,
                                        "force_igmp_version"),
                DEVINET_SYSCTL_RW_ENTRY(IGMPV2_UNSOLICITED_REPORT_INTERVAL,
                                        "igmpv2_unsolicited_report_interval"),
                DEVINET_SYSCTL_RW_ENTRY(IGMPV3_UNSOLICITED_REPORT_INTERVAL,
                                        "igmpv3_unsolicited_report_interval"),
                DEVINET_SYSCTL_RW_ENTRY(IGNORE_ROUTES_WITH_LINKDOWN,
                                        "ignore_routes_with_linkdown"),
                DEVINET_SYSCTL_RW_ENTRY(DROP_GRATUITOUS_ARP,
                                        "drop_gratuitous_arp"),

                DEVINET_SYSCTL_FLUSHING_ENTRY(NOXFRM, "disable_xfrm"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(NOPOLICY, "disable_policy"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(PROMOTE_SECONDARIES,
                                              "promote_secondaries"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(ROUTE_LOCALNET,
                                              "route_localnet"),
                DEVINET_SYSCTL_FLUSHING_ENTRY(DROP_UNICAST_IN_L2_MULTICAST,
                                              "drop_unicast_in_l2_multicast"),
        },
};

static int __devinet_sysctl_register(struct net *net, char *dev_name,
                                     int ifindex, struct ipv4_devconf *p)
{
        int i;
        struct devinet_sysctl_table *t;
        char path[sizeof("net/ipv4/conf/") + IFNAMSIZ];

        t = kmemdup(&devinet_sysctl, sizeof(*t), GFP_KERNEL_ACCOUNT);
        if (!t)
                goto out;

        for (i = 0; i < ARRAY_SIZE(t->devinet_vars); i++) {
                t->devinet_vars[i].data += (char *)p - (char *)&ipv4_devconf;
                t->devinet_vars[i].extra1 = p;
                t->devinet_vars[i].extra2 = net;
        }

        snprintf(path, sizeof(path), "net/ipv4/conf/%s", dev_name);

        t->sysctl_header = register_net_sysctl(net, path, t->devinet_vars);
        if (!t->sysctl_header)
                goto free;

        p->sysctl = t;

        inet_netconf_notify_devconf(net, RTM_NEWNETCONF, NETCONFA_ALL,
                                    ifindex, p);
        return 0;

free:
        kfree(t);
out:
        return -ENOMEM;
}

static void __devinet_sysctl_unregister(struct net *net,
                                        struct ipv4_devconf *cnf, int ifindex)
{
        struct devinet_sysctl_table *t = cnf->sysctl;

        if (t) {
                cnf->sysctl = NULL;
                unregister_net_sysctl_table(t->sysctl_header);
                kfree(t);
        }

        inet_netconf_notify_devconf(net, RTM_DELNETCONF, 0, ifindex, NULL);
}

static int devinet_sysctl_register(struct in_device *idev)
{
        int err;

        if (!sysctl_dev_name_is_allowed(idev->dev->name))
                return -EINVAL;

        err = neigh_sysctl_register(idev->dev, idev->arp_parms, NULL);
        if (err)
                return err;
        err = __devinet_sysctl_register(dev_net(idev->dev), idev->dev->name,
                                        idev->dev->ifindex, &idev->cnf);
        if (err)
                neigh_sysctl_unregister(idev->arp_parms);
        return err;
}

static void devinet_sysctl_unregister(struct in_device *idev)
{
        struct net *net = dev_net(idev->dev);

        __devinet_sysctl_unregister(net, &idev->cnf, idev->dev->ifindex);
        neigh_sysctl_unregister(idev->arp_parms);
}

static struct ctl_table ctl_forward_entry[] = {
        {
                .procname        = "ip_forward",
                .data                = &ipv4_devconf.data[
                                        IPV4_DEVCONF_FORWARDING - 1],
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = devinet_sysctl_forward,
                .extra1                = &ipv4_devconf,
                .extra2                = &init_net,
        },
};
#endif

static __net_init int devinet_init_net(struct net *net)
{
#ifdef CONFIG_SYSCTL
        struct ctl_table_header *forw_hdr;
        struct ctl_table *tbl;
#endif
        struct ipv4_devconf *all, *dflt;
        int err;
        int i;

        err = -ENOMEM;
        net->ipv4.inet_addr_lst = kmalloc_array(IN4_ADDR_HSIZE,
                                                sizeof(struct hlist_head),
                                                GFP_KERNEL);
        if (!net->ipv4.inet_addr_lst)
                goto err_alloc_hash;

        all = kmemdup(&ipv4_devconf, sizeof(ipv4_devconf), GFP_KERNEL);
        if (!all)
                goto err_alloc_all;

        dflt = kmemdup(&ipv4_devconf_dflt, sizeof(ipv4_devconf_dflt), GFP_KERNEL);
        if (!dflt)
                goto err_alloc_dflt;

#ifdef CONFIG_SYSCTL
        tbl = kmemdup(ctl_forward_entry, sizeof(ctl_forward_entry), GFP_KERNEL);
        if (!tbl)
                goto err_alloc_ctl;

        tbl[0].data = &all->data[IPV4_DEVCONF_FORWARDING - 1];
        tbl[0].extra1 = all;
        tbl[0].extra2 = net;
#endif

        if (!net_eq(net, &init_net)) {
                switch (net_inherit_devconf()) {
                case 3:
                        /* copy from the current netns */
                        memcpy(all, current->nsproxy->net_ns->ipv4.devconf_all,
                               sizeof(ipv4_devconf));
                        memcpy(dflt,
                               current->nsproxy->net_ns->ipv4.devconf_dflt,
                               sizeof(ipv4_devconf_dflt));
                        break;
                case 0:
                case 1:
                        /* copy from init_net */
                        memcpy(all, init_net.ipv4.devconf_all,
                               sizeof(ipv4_devconf));
                        memcpy(dflt, init_net.ipv4.devconf_dflt,
                               sizeof(ipv4_devconf_dflt));
                        break;
                case 2:
                        /* use compiled values */
                        break;
                }
        }

#ifdef CONFIG_SYSCTL
        err = __devinet_sysctl_register(net, "all", NETCONFA_IFINDEX_ALL, all);
        if (err < 0)
                goto err_reg_all;

        err = __devinet_sysctl_register(net, "default",
                                        NETCONFA_IFINDEX_DEFAULT, dflt);
        if (err < 0)
                goto err_reg_dflt;

        err = -ENOMEM;
        forw_hdr = register_net_sysctl_sz(net, "net/ipv4", tbl,
                                          ARRAY_SIZE(ctl_forward_entry));
        if (!forw_hdr)
                goto err_reg_ctl;
        net->ipv4.forw_hdr = forw_hdr;
#endif

        for (i = 0; i < IN4_ADDR_HSIZE; i++)
                INIT_HLIST_HEAD(&net->ipv4.inet_addr_lst[i]);

        INIT_DEFERRABLE_WORK(&net->ipv4.addr_chk_work, check_lifetime);

        net->ipv4.devconf_all = all;
        net->ipv4.devconf_dflt = dflt;
        return 0;

#ifdef CONFIG_SYSCTL
err_reg_ctl:
        __devinet_sysctl_unregister(net, dflt, NETCONFA_IFINDEX_DEFAULT);
err_reg_dflt:
        __devinet_sysctl_unregister(net, all, NETCONFA_IFINDEX_ALL);
err_reg_all:
        kfree(tbl);
err_alloc_ctl:
#endif
        kfree(dflt);
err_alloc_dflt:
        kfree(all);
err_alloc_all:
        kfree(net->ipv4.inet_addr_lst);
err_alloc_hash:
        return err;
}

static __net_exit void devinet_exit_net(struct net *net)
{
#ifdef CONFIG_SYSCTL
        const struct ctl_table *tbl;
#endif

        cancel_delayed_work_sync(&net->ipv4.addr_chk_work);

#ifdef CONFIG_SYSCTL
        tbl = net->ipv4.forw_hdr->ctl_table_arg;
        unregister_net_sysctl_table(net->ipv4.forw_hdr);
        __devinet_sysctl_unregister(net, net->ipv4.devconf_dflt,
                                    NETCONFA_IFINDEX_DEFAULT);
        __devinet_sysctl_unregister(net, net->ipv4.devconf_all,
                                    NETCONFA_IFINDEX_ALL);
        kfree(tbl);
#endif
        kfree(net->ipv4.devconf_dflt);
        kfree(net->ipv4.devconf_all);
        kfree(net->ipv4.inet_addr_lst);
}

static __net_initdata struct pernet_operations devinet_ops = {
        .init = devinet_init_net,
        .exit = devinet_exit_net,
};

static struct rtnl_af_ops inet_af_ops __read_mostly = {
        .family                  = AF_INET,
        .fill_link_af          = inet_fill_link_af,
        .get_link_af_size = inet_get_link_af_size,
        .validate_link_af = inet_validate_link_af,
        .set_link_af          = inet_set_link_af,
};

static const struct rtnl_msg_handler devinet_rtnl_msg_handlers[] __initconst = {
        {.protocol = PF_INET, .msgtype = RTM_NEWADDR, .doit = inet_rtm_newaddr,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_DELADDR, .doit = inet_rtm_deladdr,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_GETADDR, .dumpit = inet_dump_ifaddr,
         .flags = RTNL_FLAG_DUMP_UNLOCKED | RTNL_FLAG_DUMP_SPLIT_NLM_DONE},
        {.protocol = PF_INET, .msgtype = RTM_GETNETCONF,
         .doit = inet_netconf_get_devconf, .dumpit = inet_netconf_dump_devconf,
         .flags = RTNL_FLAG_DOIT_UNLOCKED | RTNL_FLAG_DUMP_UNLOCKED},
        {.owner = THIS_MODULE, .protocol = PF_INET, .msgtype = RTM_GETMULTICAST,
         .dumpit = inet_dump_ifmcaddr, .flags = RTNL_FLAG_DUMP_UNLOCKED},
};

void __init devinet_init(void)
{
        register_pernet_subsys(&devinet_ops);
        register_netdevice_notifier(&ip_netdev_notifier);

        if (rtnl_af_register(&inet_af_ops))
                panic("Unable to register inet_af_ops\n");

        rtnl_register_many(devinet_rtnl_msg_handlers);
}





















































































































    6 











    3 
    3 

    3 





































































































































































































































































































    5 





    8 






















    1 





















    5 



    1 

    3 




    1 





    2 





    6 












    6 
    6 






    6 
    3 













































    5 
    3 

    5 





















    6 


    3 
    4 




















    6 



    1 






    3 





    3 




    1 







    1 
































































































































































































































    9 




































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/pipe.c
 *
 *  Copyright (C) 1991, 1992, 1999  Linus Torvalds
 */

#include <linux/mm.h>
#include <linux/file.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/log2.h>
#include <linux/mount.h>
#include <linux/pseudo_fs.h>
#include <linux/magic.h>
#include <linux/pipe_fs_i.h>
#include <linux/uio.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/audit.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/memcontrol.h>
#include <linux/watch_queue.h>
#include <linux/sysctl.h>
#include <linux/sort.h>

#include <linux/uaccess.h>
#include <asm/ioctls.h>

#include "internal.h"

/*
 * New pipe buffers will be restricted to this size while the user is exceeding
 * their pipe buffer quota. The general pipe use case needs at least two
 * buffers: one for data yet to be read, and one for new data. If this is less
 * than two, then a write to a non-empty pipe may block even if the pipe is not
 * full. This can occur with GNU make jobserver or similar uses of pipes as
 * semaphores: multiple processes may be waiting to write tokens back to the
 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
 *
 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
 * emptied.
 */
#define PIPE_MIN_DEF_BUFFERS 2

/*
 * The max size that a non-root user is allowed to grow the pipe. Can
 * be set by root in /proc/sys/fs/pipe-max-size
 */
static unsigned int pipe_max_size = 1048576;

/* Maximum allocatable pages per user. Hard limit is unset by default, soft
 * matches default values.
 */
static unsigned long pipe_user_pages_hard;
static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR;

/*
 * We use head and tail indices that aren't masked off, except at the point of
 * dereference, but rather they're allowed to wrap naturally.  This means there
 * isn't a dead spot in the buffer, but the ring has to be a power of two and
 * <= 2^31.
 * -- David Howells 2019-09-23.
 *
 * Reads with count = 0 should always return 0.
 * -- Julian Bradfield 1999-06-07.
 *
 * FIFOs and Pipes now generate SIGIO for both readers and writers.
 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
 *
 * pipe_read & write cleanup
 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
 */

#ifdef CONFIG_PROVE_LOCKING
static int pipe_lock_cmp_fn(const struct lockdep_map *a,
                            const struct lockdep_map *b)
{
        return cmp_int((unsigned long) a, (unsigned long) b);
}
#endif

void pipe_lock(struct pipe_inode_info *pipe)
{
        if (pipe->files)
                mutex_lock(&pipe->mutex);
}
EXPORT_SYMBOL(pipe_lock);

void pipe_unlock(struct pipe_inode_info *pipe)
{
        if (pipe->files)
                mutex_unlock(&pipe->mutex);
}
EXPORT_SYMBOL(pipe_unlock);

void pipe_double_lock(struct pipe_inode_info *pipe1,
                      struct pipe_inode_info *pipe2)
{
        BUG_ON(pipe1 == pipe2);

        if (pipe1 > pipe2)
                swap(pipe1, pipe2);

        pipe_lock(pipe1);
        pipe_lock(pipe2);
}

static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe)
{
        for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
                if (pipe->tmp_page[i]) {
                        struct page *page = pipe->tmp_page[i];
                        pipe->tmp_page[i] = NULL;
                        return page;
                }
        }

        return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT);
}

static void anon_pipe_put_page(struct pipe_inode_info *pipe,
                               struct page *page)
{
        if (page_count(page) == 1) {
                for (int i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
                        if (!pipe->tmp_page[i]) {
                                pipe->tmp_page[i] = page;
                                return;
                        }
                }
        }

        put_page(page);
}

static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
                                  struct pipe_buffer *buf)
{
        struct page *page = buf->page;

        anon_pipe_put_page(pipe, page);
}

static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
                struct pipe_buffer *buf)
{
        struct page *page = buf->page;

        if (page_count(page) != 1)
                return false;
        memcg_kmem_uncharge_page(page, 0);
        __SetPageLocked(page);
        return true;
}

/**
 * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
 * @pipe:        the pipe that the buffer belongs to
 * @buf:        the buffer to attempt to steal
 *
 * Description:
 *        This function attempts to steal the &struct page attached to
 *        @buf. If successful, this function returns 0 and returns with
 *        the page locked. The caller may then reuse the page for whatever
 *        he wishes; the typical use is insertion into a different file
 *        page cache.
 */
bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
                struct pipe_buffer *buf)
{
        struct page *page = buf->page;

        /*
         * A reference of one is golden, that means that the owner of this
         * page is the only one holding a reference to it. lock the page
         * and return OK.
         */
        if (page_count(page) == 1) {
                lock_page(page);
                return true;
        }
        return false;
}
EXPORT_SYMBOL(generic_pipe_buf_try_steal);

/**
 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer
 * @pipe:        the pipe that the buffer belongs to
 * @buf:        the buffer to get a reference to
 *
 * Description:
 *        This function grabs an extra reference to @buf. It's used in
 *        the tee() system call, when we duplicate the buffers in one
 *        pipe into another.
 */
bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
{
        return try_get_page(buf->page);
}
EXPORT_SYMBOL(generic_pipe_buf_get);

/**
 * generic_pipe_buf_release - put a reference to a &struct pipe_buffer
 * @pipe:        the pipe that the buffer belongs to
 * @buf:        the buffer to put a reference to
 *
 * Description:
 *        This function releases a reference to @buf.
 */
void generic_pipe_buf_release(struct pipe_inode_info *pipe,
                              struct pipe_buffer *buf)
{
        put_page(buf->page);
}
EXPORT_SYMBOL(generic_pipe_buf_release);

static const struct pipe_buf_operations anon_pipe_buf_ops = {
        .release        = anon_pipe_buf_release,
        .try_steal        = anon_pipe_buf_try_steal,
        .get                = generic_pipe_buf_get,
};

/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
static inline bool pipe_readable(const struct pipe_inode_info *pipe)
{
        union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
        unsigned int writers = READ_ONCE(pipe->writers);

        return !pipe_empty(idx.head, idx.tail) || !writers;
}

static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe,
                                            struct pipe_buffer *buf,
                                            unsigned int tail)
{
        pipe_buf_release(pipe, buf);

        /*
         * If the pipe has a watch_queue, we need additional protection
         * by the spinlock because notifications get posted with only
         * this spinlock, no mutex
         */
        if (pipe_has_watch_queue(pipe)) {
                spin_lock_irq(&pipe->rd_wait.lock);
#ifdef CONFIG_WATCH_QUEUE
                if (buf->flags & PIPE_BUF_FLAG_LOSS)
                        pipe->note_loss = true;
#endif
                pipe->tail = ++tail;
                spin_unlock_irq(&pipe->rd_wait.lock);
                return tail;
        }

        /*
         * Without a watch_queue, we can simply increment the tail
         * without the spinlock - the mutex is enough.
         */
        pipe->tail = ++tail;
        return tail;
}

static ssize_t
anon_pipe_read(struct kiocb *iocb, struct iov_iter *to)
{
        size_t total_len = iov_iter_count(to);
        struct file *filp = iocb->ki_filp;
        struct pipe_inode_info *pipe = filp->private_data;
        bool wake_writer = false, wake_next_reader = false;
        ssize_t ret;

        /* Null read succeeds. */
        if (unlikely(total_len == 0))
                return 0;

        ret = 0;
        mutex_lock(&pipe->mutex);

        /*
         * We only wake up writers if the pipe was full when we started reading
         * and it is no longer full after reading to avoid unnecessary wakeups.
         *
         * But when we do wake up writers, we do so using a sync wakeup
         * (WF_SYNC), because we want them to get going and generate more
         * data for us.
         */
        for (;;) {
                /* Read ->head with a barrier vs post_one_notification() */
                unsigned int head = smp_load_acquire(&pipe->head);
                unsigned int tail = pipe->tail;

#ifdef CONFIG_WATCH_QUEUE
                if (pipe->note_loss) {
                        struct watch_notification n;

                        if (total_len < 8) {
                                if (ret == 0)
                                        ret = -ENOBUFS;
                                break;
                        }

                        n.type = WATCH_TYPE_META;
                        n.subtype = WATCH_META_LOSS_NOTIFICATION;
                        n.info = watch_sizeof(n);
                        if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) {
                                if (ret == 0)
                                        ret = -EFAULT;
                                break;
                        }
                        ret += sizeof(n);
                        total_len -= sizeof(n);
                        pipe->note_loss = false;
                }
#endif

                if (!pipe_empty(head, tail)) {
                        struct pipe_buffer *buf = pipe_buf(pipe, tail);
                        size_t chars = buf->len;
                        size_t written;
                        int error;

                        if (chars > total_len) {
                                if (buf->flags & PIPE_BUF_FLAG_WHOLE) {
                                        if (ret == 0)
                                                ret = -ENOBUFS;
                                        break;
                                }
                                chars = total_len;
                        }

                        error = pipe_buf_confirm(pipe, buf);
                        if (error) {
                                if (!ret)
                                        ret = error;
                                break;
                        }

                        written = copy_page_to_iter(buf->page, buf->offset, chars, to);
                        if (unlikely(written < chars)) {
                                if (!ret)
                                        ret = -EFAULT;
                                break;
                        }
                        ret += chars;
                        buf->offset += chars;
                        buf->len -= chars;

                        /* Was it a packet buffer? Clean up and exit */
                        if (buf->flags & PIPE_BUF_FLAG_PACKET) {
                                total_len = chars;
                                buf->len = 0;
                        }

                        if (!buf->len) {
                                wake_writer |= pipe_full(head, tail, pipe->max_usage);
                                tail = pipe_update_tail(pipe, buf, tail);
                        }
                        total_len -= chars;
                        if (!total_len)
                                break;        /* common path: read succeeded */
                        if (!pipe_empty(head, tail))        /* More to do? */
                                continue;
                }

                if (!pipe->writers)
                        break;
                if (ret)
                        break;
                if ((filp->f_flags & O_NONBLOCK) ||
                    (iocb->ki_flags & IOCB_NOWAIT)) {
                        ret = -EAGAIN;
                        break;
                }
                mutex_unlock(&pipe->mutex);
                /*
                 * We only get here if we didn't actually read anything.
                 *
                 * But because we didn't read anything, at this point we can
                 * just return directly with -ERESTARTSYS if we're interrupted,
                 * since we've done any required wakeups and there's no need
                 * to mark anything accessed. And we've dropped the lock.
                 */
                if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
                        return -ERESTARTSYS;

                wake_next_reader = true;
                mutex_lock(&pipe->mutex);
        }
        if (pipe_is_empty(pipe))
                wake_next_reader = false;
        mutex_unlock(&pipe->mutex);

        if (wake_writer)
                wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
        if (wake_next_reader)
                wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
        kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
        return ret;
}

static ssize_t
fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to)
{
        int ret = anon_pipe_read(iocb, to);
        if (ret > 0)
                file_accessed(iocb->ki_filp);
        return ret;
}

static inline int is_packetized(struct file *file)
{
        return (file->f_flags & O_DIRECT) != 0;
}

/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
static inline bool pipe_writable(const struct pipe_inode_info *pipe)
{
        union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) };
        unsigned int max_usage = READ_ONCE(pipe->max_usage);

        return !pipe_full(idx.head, idx.tail, max_usage) ||
                !READ_ONCE(pipe->readers);
}

static ssize_t
anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
{
        struct file *filp = iocb->ki_filp;
        struct pipe_inode_info *pipe = filp->private_data;
        unsigned int head;
        ssize_t ret = 0;
        size_t total_len = iov_iter_count(from);
        ssize_t chars;
        bool was_empty = false;
        bool wake_next_writer = false;

        /*
         * Reject writing to watch queue pipes before the point where we lock
         * the pipe.
         * Otherwise, lockdep would be unhappy if the caller already has another
         * pipe locked.
         * If we had to support locking a normal pipe and a notification pipe at
         * the same time, we could set up lockdep annotations for that, but
         * since we don't actually need that, it's simpler to just bail here.
         */
        if (pipe_has_watch_queue(pipe))
                return -EXDEV;

        /* Null write succeeds. */
        if (unlikely(total_len == 0))
                return 0;

        mutex_lock(&pipe->mutex);

        if (!pipe->readers) {
                send_sig(SIGPIPE, current, 0);
                ret = -EPIPE;
                goto out;
        }

        /*
         * If it wasn't empty we try to merge new data into
         * the last buffer.
         *
         * That naturally merges small writes, but it also
         * page-aligns the rest of the writes for large writes
         * spanning multiple pages.
         */
        head = pipe->head;
        was_empty = pipe_empty(head, pipe->tail);
        chars = total_len & (PAGE_SIZE-1);
        if (chars && !was_empty) {
                struct pipe_buffer *buf = pipe_buf(pipe, head - 1);
                int offset = buf->offset + buf->len;

                if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
                    offset + chars <= PAGE_SIZE) {
                        ret = pipe_buf_confirm(pipe, buf);
                        if (ret)
                                goto out;

                        ret = copy_page_from_iter(buf->page, offset, chars, from);
                        if (unlikely(ret < chars)) {
                                ret = -EFAULT;
                                goto out;
                        }

                        buf->len += ret;
                        if (!iov_iter_count(from))
                                goto out;
                }
        }

        for (;;) {
                if (!pipe->readers) {
                        send_sig(SIGPIPE, current, 0);
                        if (!ret)
                                ret = -EPIPE;
                        break;
                }

                head = pipe->head;
                if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
                        struct pipe_buffer *buf;
                        struct page *page;
                        int copied;

                        page = anon_pipe_get_page(pipe);
                        if (unlikely(!page)) {
                                if (!ret)
                                        ret = -ENOMEM;
                                break;
                        }

                        copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
                        if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) {
                                anon_pipe_put_page(pipe, page);
                                if (!ret)
                                        ret = -EFAULT;
                                break;
                        }

                        pipe->head = head + 1;
                        /* Insert it into the buffer array */
                        buf = pipe_buf(pipe, head);
                        buf->page = page;
                        buf->ops = &anon_pipe_buf_ops;
                        buf->offset = 0;
                        if (is_packetized(filp))
                                buf->flags = PIPE_BUF_FLAG_PACKET;
                        else
                                buf->flags = PIPE_BUF_FLAG_CAN_MERGE;

                        buf->len = copied;
                        ret += copied;

                        if (!iov_iter_count(from))
                                break;

                        continue;
                }

                /* Wait for buffer space to become available. */
                if ((filp->f_flags & O_NONBLOCK) ||
                    (iocb->ki_flags & IOCB_NOWAIT)) {
                        if (!ret)
                                ret = -EAGAIN;
                        break;
                }
                if (signal_pending(current)) {
                        if (!ret)
                                ret = -ERESTARTSYS;
                        break;
                }

                /*
                 * We're going to release the pipe lock and wait for more
                 * space. We wake up any readers if necessary, and then
                 * after waiting we need to re-check whether the pipe
                 * become empty while we dropped the lock.
                 */
                mutex_unlock(&pipe->mutex);
                if (was_empty)
                        wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
                kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
                wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
                mutex_lock(&pipe->mutex);
                was_empty = pipe_is_empty(pipe);
                wake_next_writer = true;
        }
out:
        if (pipe_is_full(pipe))
                wake_next_writer = false;
        mutex_unlock(&pipe->mutex);

        /*
         * If we do do a wakeup event, we do a 'sync' wakeup, because we
         * want the reader to start processing things asap, rather than
         * leave the data pending.
         *
         * This is particularly important for small writes, because of
         * how (for example) the GNU make jobserver uses small writes to
         * wake up pending jobs
         *
         * Epoll nonsensically wants a wakeup whether the pipe
         * was already empty or not.
         */
        if (was_empty || pipe->poll_usage)
                wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
        kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
        if (wake_next_writer)
                wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
        return ret;
}

static ssize_t
fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from)
{
        int ret = anon_pipe_write(iocb, from);
        if (ret > 0) {
                struct file *filp = iocb->ki_filp;
                if (sb_start_write_trylock(file_inode(filp)->i_sb)) {
                        int err = file_update_time(filp);
                        if (err)
                                ret = err;
                        sb_end_write(file_inode(filp)->i_sb);
                }
        }
        return ret;
}

static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
        struct pipe_inode_info *pipe = filp->private_data;
        unsigned int count, head, tail;

        switch (cmd) {
        case FIONREAD:
                mutex_lock(&pipe->mutex);
                count = 0;
                head = pipe->head;
                tail = pipe->tail;

                while (!pipe_empty(head, tail)) {
                        count += pipe_buf(pipe, tail)->len;
                        tail++;
                }
                mutex_unlock(&pipe->mutex);

                return put_user(count, (int __user *)arg);

#ifdef CONFIG_WATCH_QUEUE
        case IOC_WATCH_QUEUE_SET_SIZE: {
                int ret;
                mutex_lock(&pipe->mutex);
                ret = watch_queue_set_size(pipe, arg);
                mutex_unlock(&pipe->mutex);
                return ret;
        }

        case IOC_WATCH_QUEUE_SET_FILTER:
                return watch_queue_set_filter(
                        pipe, (struct watch_notification_filter __user *)arg);
#endif

        default:
                return -ENOIOCTLCMD;
        }
}

/* No kernel lock held - fine */
static __poll_t
pipe_poll(struct file *filp, poll_table *wait)
{
        __poll_t mask;
        struct pipe_inode_info *pipe = filp->private_data;
        union pipe_index idx;

        /* Epoll has some historical nasty semantics, this enables them */
        WRITE_ONCE(pipe->poll_usage, true);

        /*
         * Reading pipe state only -- no need for acquiring the semaphore.
         *
         * But because this is racy, the code has to add the
         * entry to the poll table _first_ ..
         */
        if (filp->f_mode & FMODE_READ)
                poll_wait(filp, &pipe->rd_wait, wait);
        if (filp->f_mode & FMODE_WRITE)
                poll_wait(filp, &pipe->wr_wait, wait);

        /*
         * .. and only then can you do the racy tests. That way,
         * if something changes and you got it wrong, the poll
         * table entry will wake you up and fix it.
         */
        idx.head_tail = READ_ONCE(pipe->head_tail);

        mask = 0;
        if (filp->f_mode & FMODE_READ) {
                if (!pipe_empty(idx.head, idx.tail))
                        mask |= EPOLLIN | EPOLLRDNORM;
                if (!pipe->writers && filp->f_pipe != pipe->w_counter)
                        mask |= EPOLLHUP;
        }

        if (filp->f_mode & FMODE_WRITE) {
                if (!pipe_full(idx.head, idx.tail, pipe->max_usage))
                        mask |= EPOLLOUT | EPOLLWRNORM;
                /*
                 * Most Unices do not set EPOLLERR for FIFOs but on Linux they
                 * behave exactly like pipes for poll().
                 */
                if (!pipe->readers)
                        mask |= EPOLLERR;
        }

        return mask;
}

static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
{
        int kill = 0;

        spin_lock(&inode->i_lock);
        if (!--pipe->files) {
                inode->i_pipe = NULL;
                kill = 1;
        }
        spin_unlock(&inode->i_lock);

        if (kill)
                free_pipe_info(pipe);
}

static int
pipe_release(struct inode *inode, struct file *file)
{
        struct pipe_inode_info *pipe = file->private_data;

        mutex_lock(&pipe->mutex);
        if (file->f_mode & FMODE_READ)
                pipe->readers--;
        if (file->f_mode & FMODE_WRITE)
                pipe->writers--;

        /* Was that the last reader or writer, but not the other side? */
        if (!pipe->readers != !pipe->writers) {
                wake_up_interruptible_all(&pipe->rd_wait);
                wake_up_interruptible_all(&pipe->wr_wait);
                kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
                kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
        }
        mutex_unlock(&pipe->mutex);

        put_pipe_info(inode, pipe);
        return 0;
}

static int
pipe_fasync(int fd, struct file *filp, int on)
{
        struct pipe_inode_info *pipe = filp->private_data;
        int retval = 0;

        mutex_lock(&pipe->mutex);
        if (filp->f_mode & FMODE_READ)
                retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
        if ((filp->f_mode & FMODE_WRITE) && retval >= 0) {
                retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
                if (retval < 0 && (filp->f_mode & FMODE_READ))
                        /* this can happen only if on == T */
                        fasync_helper(-1, filp, 0, &pipe->fasync_readers);
        }
        mutex_unlock(&pipe->mutex);
        return retval;
}

unsigned long account_pipe_buffers(struct user_struct *user,
                                   unsigned long old, unsigned long new)
{
        return atomic_long_add_return(new - old, &user->pipe_bufs);
}

bool too_many_pipe_buffers_soft(unsigned long user_bufs)
{
        unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft);

        return soft_limit && user_bufs > soft_limit;
}

bool too_many_pipe_buffers_hard(unsigned long user_bufs)
{
        unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard);

        return hard_limit && user_bufs > hard_limit;
}

bool pipe_is_unprivileged_user(void)
{
        return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN);
}

struct pipe_inode_info *alloc_pipe_info(void)
{
        struct pipe_inode_info *pipe;
        unsigned long pipe_bufs = PIPE_DEF_BUFFERS;
        struct user_struct *user = get_current_user();
        unsigned long user_bufs;
        unsigned int max_size = READ_ONCE(pipe_max_size);

        pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
        if (pipe == NULL)
                goto out_free_uid;

        if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE))
                pipe_bufs = max_size >> PAGE_SHIFT;

        user_bufs = account_pipe_buffers(user, 0, pipe_bufs);

        if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) {
                user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS);
                pipe_bufs = PIPE_MIN_DEF_BUFFERS;
        }

        if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user())
                goto out_revert_acct;

        pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
                             GFP_KERNEL_ACCOUNT);

        if (pipe->bufs) {
                init_waitqueue_head(&pipe->rd_wait);
                init_waitqueue_head(&pipe->wr_wait);
                pipe->r_counter = pipe->w_counter = 1;
                pipe->max_usage = pipe_bufs;
                pipe->ring_size = pipe_bufs;
                pipe->nr_accounted = pipe_bufs;
                pipe->user = user;
                mutex_init(&pipe->mutex);
                lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL);
                return pipe;
        }

out_revert_acct:
        (void) account_pipe_buffers(user, pipe_bufs, 0);
        kfree(pipe);
out_free_uid:
        free_uid(user);
        return NULL;
}

void free_pipe_info(struct pipe_inode_info *pipe)
{
        unsigned int i;

#ifdef CONFIG_WATCH_QUEUE
        if (pipe->watch_queue)
                watch_queue_clear(pipe->watch_queue);
#endif

        (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
        free_uid(pipe->user);
        for (i = 0; i < pipe->ring_size; i++) {
                struct pipe_buffer *buf = pipe->bufs + i;
                if (buf->ops)
                        pipe_buf_release(pipe, buf);
        }
#ifdef CONFIG_WATCH_QUEUE
        if (pipe->watch_queue)
                put_watch_queue(pipe->watch_queue);
#endif
        for (i = 0; i < ARRAY_SIZE(pipe->tmp_page); i++) {
                if (pipe->tmp_page[i])
                        __free_page(pipe->tmp_page[i]);
        }
        kfree(pipe->bufs);
        kfree(pipe);
}

static struct vfsmount *pipe_mnt __ro_after_init;

/*
 * pipefs_dname() is called from d_path().
 */
static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
{
        return dynamic_dname(buffer, buflen, "pipe:[%lu]",
                                d_inode(dentry)->i_ino);
}

static const struct dentry_operations pipefs_dentry_operations = {
        .d_dname        = pipefs_dname,
};

static const struct file_operations pipeanon_fops;

static struct inode * get_pipe_inode(void)
{
        struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb);
        struct pipe_inode_info *pipe;

        if (!inode)
                goto fail_inode;

        inode->i_ino = get_next_ino();

        pipe = alloc_pipe_info();
        if (!pipe)
                goto fail_iput;

        inode->i_pipe = pipe;
        pipe->files = 2;
        pipe->readers = pipe->writers = 1;
        inode->i_fop = &pipeanon_fops;

        /*
         * Mark the inode dirty from the very beginning,
         * that way it will never be moved to the dirty
         * list because "mark_inode_dirty()" will think
         * that it already _is_ on the dirty list.
         */
        inode->i_state = I_DIRTY;
        inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
        inode->i_uid = current_fsuid();
        inode->i_gid = current_fsgid();
        simple_inode_init_ts(inode);

        return inode;

fail_iput:
        iput(inode);

fail_inode:
        return NULL;
}

int create_pipe_files(struct file **res, int flags)
{
        struct inode *inode = get_pipe_inode();
        struct file *f;
        int error;

        if (!inode)
                return -ENFILE;

        if (flags & O_NOTIFICATION_PIPE) {
                error = watch_queue_init(inode->i_pipe);
                if (error) {
                        free_pipe_info(inode->i_pipe);
                        iput(inode);
                        return error;
                }
        }

        f = alloc_file_pseudo(inode, pipe_mnt, "",
                                O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)),
                                &pipeanon_fops);
        if (IS_ERR(f)) {
                free_pipe_info(inode->i_pipe);
                iput(inode);
                return PTR_ERR(f);
        }

        f->private_data = inode->i_pipe;
        f->f_pipe = 0;

        res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK),
                                  &pipeanon_fops);
        if (IS_ERR(res[0])) {
                put_pipe_info(inode, inode->i_pipe);
                fput(f);
                return PTR_ERR(res[0]);
        }
        res[0]->private_data = inode->i_pipe;
        res[0]->f_pipe = 0;
        res[1] = f;
        stream_open(inode, res[0]);
        stream_open(inode, res[1]);
        /*
         * Disable permission and pre-content events, but enable legacy
         * inotify events for legacy users.
         */
        file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM);
        file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM);
        return 0;
}

static int __do_pipe_flags(int *fd, struct file **files, int flags)
{
        int error;
        int fdw, fdr;

        if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
                return -EINVAL;

        error = create_pipe_files(files, flags);
        if (error)
                return error;

        error = get_unused_fd_flags(flags);
        if (error < 0)
                goto err_read_pipe;
        fdr = error;

        error = get_unused_fd_flags(flags);
        if (error < 0)
                goto err_fdr;
        fdw = error;

        audit_fd_pair(fdr, fdw);
        fd[0] = fdr;
        fd[1] = fdw;
        /* pipe groks IOCB_NOWAIT */
        files[0]->f_mode |= FMODE_NOWAIT;
        files[1]->f_mode |= FMODE_NOWAIT;
        return 0;

 err_fdr:
        put_unused_fd(fdr);
 err_read_pipe:
        fput(files[0]);
        fput(files[1]);
        return error;
}

int do_pipe_flags(int *fd, int flags)
{
        struct file *files[2];
        int error = __do_pipe_flags(fd, files, flags);
        if (!error) {
                fd_install(fd[0], files[0]);
                fd_install(fd[1], files[1]);
        }
        return error;
}

/*
 * sys_pipe() is the normal C calling standard for creating
 * a pipe. It's not the way Unix traditionally does this, though.
 */
static int do_pipe2(int __user *fildes, int flags)
{
        struct file *files[2];
        int fd[2];
        int error;

        error = __do_pipe_flags(fd, files, flags);
        if (!error) {
                if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) {
                        fput(files[0]);
                        fput(files[1]);
                        put_unused_fd(fd[0]);
                        put_unused_fd(fd[1]);
                        error = -EFAULT;
                } else {
                        fd_install(fd[0], files[0]);
                        fd_install(fd[1], files[1]);
                }
        }
        return error;
}

SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags)
{
        return do_pipe2(fildes, flags);
}

SYSCALL_DEFINE1(pipe, int __user *, fildes)
{
        return do_pipe2(fildes, 0);
}

/*
 * This is the stupid "wait for pipe to be readable or writable"
 * model.
 *
 * See pipe_read/write() for the proper kind of exclusive wait,
 * but that requires that we wake up any other readers/writers
 * if we then do not end up reading everything (ie the whole
 * "wake_next_reader/writer" logic in pipe_read/write()).
 */
void pipe_wait_readable(struct pipe_inode_info *pipe)
{
        pipe_unlock(pipe);
        wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
        pipe_lock(pipe);
}

void pipe_wait_writable(struct pipe_inode_info *pipe)
{
        pipe_unlock(pipe);
        wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
        pipe_lock(pipe);
}

/*
 * This depends on both the wait (here) and the wakeup (wake_up_partner)
 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
 * race with the count check and waitqueue prep.
 *
 * Normally in order to avoid races, you'd do the prepare_to_wait() first,
 * then check the condition you're waiting for, and only then sleep. But
 * because of the pipe lock, we can check the condition before being on
 * the wait queue.
 *
 * We use the 'rd_wait' waitqueue for pipe partner waiting.
 */
static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
{
        DEFINE_WAIT(rdwait);
        int cur = *cnt;

        while (cur == *cnt) {
                prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
                pipe_unlock(pipe);
                schedule();
                finish_wait(&pipe->rd_wait, &rdwait);
                pipe_lock(pipe);
                if (signal_pending(current))
                        break;
        }
        return cur == *cnt ? -ERESTARTSYS : 0;
}

static void wake_up_partner(struct pipe_inode_info *pipe)
{
        wake_up_interruptible_all(&pipe->rd_wait);
}

static int fifo_open(struct inode *inode, struct file *filp)
{
        bool is_pipe = inode->i_fop == &pipeanon_fops;
        struct pipe_inode_info *pipe;
        int ret;

        filp->f_pipe = 0;

        spin_lock(&inode->i_lock);
        if (inode->i_pipe) {
                pipe = inode->i_pipe;
                pipe->files++;
                spin_unlock(&inode->i_lock);
        } else {
                spin_unlock(&inode->i_lock);
                pipe = alloc_pipe_info();
                if (!pipe)
                        return -ENOMEM;
                pipe->files = 1;
                spin_lock(&inode->i_lock);
                if (unlikely(inode->i_pipe)) {
                        inode->i_pipe->files++;
                        spin_unlock(&inode->i_lock);
                        free_pipe_info(pipe);
                        pipe = inode->i_pipe;
                } else {
                        inode->i_pipe = pipe;
                        spin_unlock(&inode->i_lock);
                }
        }
        filp->private_data = pipe;
        /* OK, we have a pipe and it's pinned down */

        mutex_lock(&pipe->mutex);

        /* We can only do regular read/write on fifos */
        stream_open(inode, filp);

        switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) {
        case FMODE_READ:
        /*
         *  O_RDONLY
         *  POSIX.1 says that O_NONBLOCK means return with the FIFO
         *  opened, even when there is no process writing the FIFO.
         */
                pipe->r_counter++;
                if (pipe->readers++ == 0)
                        wake_up_partner(pipe);

                if (!is_pipe && !pipe->writers) {
                        if ((filp->f_flags & O_NONBLOCK)) {
                                /* suppress EPOLLHUP until we have
                                 * seen a writer */
                                filp->f_pipe = pipe->w_counter;
                        } else {
                                if (wait_for_partner(pipe, &pipe->w_counter))
                                        goto err_rd;
                        }
                }
                break;

        case FMODE_WRITE:
        /*
         *  O_WRONLY
         *  POSIX.1 says that O_NONBLOCK means return -1 with
         *  errno=ENXIO when there is no process reading the FIFO.
         */
                ret = -ENXIO;
                if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
                        goto err;

                pipe->w_counter++;
                if (!pipe->writers++)
                        wake_up_partner(pipe);

                if (!is_pipe && !pipe->readers) {
                        if (wait_for_partner(pipe, &pipe->r_counter))
                                goto err_wr;
                }
                break;

        case FMODE_READ | FMODE_WRITE:
        /*
         *  O_RDWR
         *  POSIX.1 leaves this case "undefined" when O_NONBLOCK is set.
         *  This implementation will NEVER block on a O_RDWR open, since
         *  the process can at least talk to itself.
         */

                pipe->readers++;
                pipe->writers++;
                pipe->r_counter++;
                pipe->w_counter++;
                if (pipe->readers == 1 || pipe->writers == 1)
                        wake_up_partner(pipe);
                break;

        default:
                ret = -EINVAL;
                goto err;
        }

        /* Ok! */
        mutex_unlock(&pipe->mutex);
        return 0;

err_rd:
        if (!--pipe->readers)
                wake_up_interruptible(&pipe->wr_wait);
        ret = -ERESTARTSYS;
        goto err;

err_wr:
        if (!--pipe->writers)
                wake_up_interruptible_all(&pipe->rd_wait);
        ret = -ERESTARTSYS;
        goto err;

err:
        mutex_unlock(&pipe->mutex);

        put_pipe_info(inode, pipe);
        return ret;
}

const struct file_operations pipefifo_fops = {
        .open                = fifo_open,
        .read_iter        = fifo_pipe_read,
        .write_iter        = fifo_pipe_write,
        .poll                = pipe_poll,
        .unlocked_ioctl        = pipe_ioctl,
        .release        = pipe_release,
        .fasync                = pipe_fasync,
        .splice_write        = iter_file_splice_write,
};

static const struct file_operations pipeanon_fops = {
        .open                = fifo_open,
        .read_iter        = anon_pipe_read,
        .write_iter        = anon_pipe_write,
        .poll                = pipe_poll,
        .unlocked_ioctl        = pipe_ioctl,
        .release        = pipe_release,
        .fasync                = pipe_fasync,
        .splice_write        = iter_file_splice_write,
};

/*
 * Currently we rely on the pipe array holding a power-of-2 number
 * of pages. Returns 0 on error.
 */
unsigned int round_pipe_size(unsigned int size)
{
        if (size > (1U << 31))
                return 0;

        /* Minimum pipe size, as required by POSIX */
        if (size < PAGE_SIZE)
                return PAGE_SIZE;

        return roundup_pow_of_two(size);
}

/*
 * Resize the pipe ring to a number of slots.
 *
 * Note the pipe can be reduced in capacity, but only if the current
 * occupancy doesn't exceed nr_slots; if it does, EBUSY will be
 * returned instead.
 */
int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
{
        struct pipe_buffer *bufs;
        unsigned int head, tail, mask, n;

        /* nr_slots larger than limits of pipe->{head,tail} */
        if (unlikely(nr_slots > (pipe_index_t)-1u))
                return -EINVAL;

        bufs = kcalloc(nr_slots, sizeof(*bufs),
                       GFP_KERNEL_ACCOUNT | __GFP_NOWARN);
        if (unlikely(!bufs))
                return -ENOMEM;

        spin_lock_irq(&pipe->rd_wait.lock);
        mask = pipe->ring_size - 1;
        head = pipe->head;
        tail = pipe->tail;

        n = pipe_occupancy(head, tail);
        if (nr_slots < n) {
                spin_unlock_irq(&pipe->rd_wait.lock);
                kfree(bufs);
                return -EBUSY;
        }

        /*
         * The pipe array wraps around, so just start the new one at zero
         * and adjust the indices.
         */
        if (n > 0) {
                unsigned int h = head & mask;
                unsigned int t = tail & mask;
                if (h > t) {
                        memcpy(bufs, pipe->bufs + t,
                               n * sizeof(struct pipe_buffer));
                } else {
                        unsigned int tsize = pipe->ring_size - t;
                        if (h > 0)
                                memcpy(bufs + tsize, pipe->bufs,
                                       h * sizeof(struct pipe_buffer));
                        memcpy(bufs, pipe->bufs + t,
                               tsize * sizeof(struct pipe_buffer));
                }
        }

        head = n;
        tail = 0;

        kfree(pipe->bufs);
        pipe->bufs = bufs;
        pipe->ring_size = nr_slots;
        if (pipe->max_usage > nr_slots)
                pipe->max_usage = nr_slots;
        pipe->tail = tail;
        pipe->head = head;

        if (!pipe_has_watch_queue(pipe)) {
                pipe->max_usage = nr_slots;
                pipe->nr_accounted = nr_slots;
        }

        spin_unlock_irq(&pipe->rd_wait.lock);

        /* This might have made more room for writers */
        wake_up_interruptible(&pipe->wr_wait);
        return 0;
}

/*
 * Allocate a new array of pipe buffers and copy the info over. Returns the
 * pipe size if successful, or return -ERROR on error.
 */
static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)
{
        unsigned long user_bufs;
        unsigned int nr_slots, size;
        long ret = 0;

        if (pipe_has_watch_queue(pipe))
                return -EBUSY;

        size = round_pipe_size(arg);
        nr_slots = size >> PAGE_SHIFT;

        if (!nr_slots)
                return -EINVAL;

        /*
         * If trying to increase the pipe capacity, check that an
         * unprivileged user is not trying to exceed various limits
         * (soft limit check here, hard limit check just below).
         * Decreasing the pipe capacity is always permitted, even
         * if the user is currently over a limit.
         */
        if (nr_slots > pipe->max_usage &&
                        size > pipe_max_size && !capable(CAP_SYS_RESOURCE))
                return -EPERM;

        user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);

        if (nr_slots > pipe->max_usage &&
                        (too_many_pipe_buffers_hard(user_bufs) ||
                         too_many_pipe_buffers_soft(user_bufs)) &&
                        pipe_is_unprivileged_user()) {
                ret = -EPERM;
                goto out_revert_acct;
        }

        ret = pipe_resize_ring(pipe, nr_slots);
        if (ret < 0)
                goto out_revert_acct;

        return pipe->max_usage * PAGE_SIZE;

out_revert_acct:
        (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
        return ret;
}

/*
 * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is
 * not enough to verify that this is a pipe.
 */
struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice)
{
        struct pipe_inode_info *pipe = file->private_data;

        if (!pipe)
                return NULL;
        if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops)
                return NULL;
        if (for_splice && pipe_has_watch_queue(pipe))
                return NULL;
        return pipe;
}

long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg)
{
        struct pipe_inode_info *pipe;
        long ret;

        pipe = get_pipe_info(file, false);
        if (!pipe)
                return -EBADF;

        mutex_lock(&pipe->mutex);

        switch (cmd) {
        case F_SETPIPE_SZ:
                ret = pipe_set_size(pipe, arg);
                break;
        case F_GETPIPE_SZ:
                ret = pipe->max_usage * PAGE_SIZE;
                break;
        default:
                ret = -EINVAL;
                break;
        }

        mutex_unlock(&pipe->mutex);
        return ret;
}

static const struct super_operations pipefs_ops = {
        .destroy_inode = free_inode_nonrcu,
        .statfs = simple_statfs,
};

/*
 * pipefs should _never_ be mounted by userland - too much of security hassle,
 * no real gain from having the whole file system mounted. So we don't need
 * any operations on the root directory. However, we need a non-trivial
 * d_name - pipe: will go nicely and kill the special-casing in procfs.
 */

static int pipefs_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC);
        if (!ctx)
                return -ENOMEM;
        ctx->ops = &pipefs_ops;
        ctx->dops = &pipefs_dentry_operations;
        return 0;
}

static struct file_system_type pipe_fs_type = {
        .name                = "pipefs",
        .init_fs_context = pipefs_init_fs_context,
        .kill_sb        = kill_anon_super,
};

#ifdef CONFIG_SYSCTL
static int do_proc_dopipe_max_size_conv(unsigned long *lvalp,
                                        unsigned int *valp,
                                        int write, void *data)
{
        if (write) {
                unsigned int val;

                val = round_pipe_size(*lvalp);
                if (val == 0)
                        return -EINVAL;

                *valp = val;
        } else {
                unsigned int val = *valp;
                *lvalp = (unsigned long) val;
        }

        return 0;
}

static int proc_dopipe_max_size(const struct ctl_table *table, int write,
                                void *buffer, size_t *lenp, loff_t *ppos)
{
        return do_proc_douintvec(table, write, buffer, lenp, ppos,
                                 do_proc_dopipe_max_size_conv, NULL);
}

static const struct ctl_table fs_pipe_sysctls[] = {
        {
                .procname        = "pipe-max-size",
                .data                = &pipe_max_size,
                .maxlen                = sizeof(pipe_max_size),
                .mode                = 0644,
                .proc_handler        = proc_dopipe_max_size,
        },
        {
                .procname        = "pipe-user-pages-hard",
                .data                = &pipe_user_pages_hard,
                .maxlen                = sizeof(pipe_user_pages_hard),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
        {
                .procname        = "pipe-user-pages-soft",
                .data                = &pipe_user_pages_soft,
                .maxlen                = sizeof(pipe_user_pages_soft),
                .mode                = 0644,
                .proc_handler        = proc_doulongvec_minmax,
        },
};
#endif

static int __init init_pipe_fs(void)
{
        int err = register_filesystem(&pipe_fs_type);

        if (!err) {
                pipe_mnt = kern_mount(&pipe_fs_type);
                if (IS_ERR(pipe_mnt)) {
                        err = PTR_ERR(pipe_mnt);
                        unregister_filesystem(&pipe_fs_type);
                }
        }
#ifdef CONFIG_SYSCTL
        register_sysctl_init("fs", fs_pipe_sysctls);
#endif
        return err;
}

fs_initcall(init_pipe_fs);






















































































































































































































































































































































































































   29 

   28 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
// SPDX-License-Identifier: GPL-2.0-only
/*
 * mm/percpu-vm.c - vmalloc area based chunk allocation
 *
 * Copyright (C) 2010                SUSE Linux Products GmbH
 * Copyright (C) 2010                Tejun Heo <tj@kernel.org>
 *
 * Chunks are mapped into vmalloc areas and populated page by page.
 * This is the default chunk allocator.
 */
#include "internal.h"

static struct page *pcpu_chunk_page(struct pcpu_chunk *chunk,
                                    unsigned int cpu, int page_idx)
{
        /* must not be used on pre-mapped chunk */
        WARN_ON(chunk->immutable);

        return vmalloc_to_page((void *)pcpu_chunk_addr(chunk, cpu, page_idx));
}

/**
 * pcpu_get_pages - get temp pages array
 *
 * Returns pointer to array of pointers to struct page which can be indexed
 * with pcpu_page_idx().  Note that there is only one array and accesses
 * should be serialized by pcpu_alloc_mutex.
 *
 * RETURNS:
 * Pointer to temp pages array on success.
 */
static struct page **pcpu_get_pages(void)
{
        static struct page **pages;
        size_t pages_size = pcpu_nr_units * pcpu_unit_pages * sizeof(pages[0]);

        lockdep_assert_held(&pcpu_alloc_mutex);

        if (!pages)
                pages = pcpu_mem_zalloc(pages_size, GFP_KERNEL);
        return pages;
}

/**
 * pcpu_free_pages - free pages which were allocated for @chunk
 * @chunk: chunk pages were allocated for
 * @pages: array of pages to be freed, indexed by pcpu_page_idx()
 * @page_start: page index of the first page to be freed
 * @page_end: page index of the last page to be freed + 1
 *
 * Free pages [@page_start and @page_end) in @pages for all units.
 * The pages were allocated for @chunk.
 */
static void pcpu_free_pages(struct pcpu_chunk *chunk,
                            struct page **pages, int page_start, int page_end)
{
        unsigned int cpu;
        int i;

        for_each_possible_cpu(cpu) {
                for (i = page_start; i < page_end; i++) {
                        struct page *page = pages[pcpu_page_idx(cpu, i)];

                        if (page)
                                __free_page(page);
                }
        }
}

/**
 * pcpu_alloc_pages - allocates pages for @chunk
 * @chunk: target chunk
 * @pages: array to put the allocated pages into, indexed by pcpu_page_idx()
 * @page_start: page index of the first page to be allocated
 * @page_end: page index of the last page to be allocated + 1
 * @gfp: allocation flags passed to the underlying allocator
 *
 * Allocate pages [@page_start,@page_end) into @pages for all units.
 * The allocation is for @chunk.  Percpu core doesn't care about the
 * content of @pages and will pass it verbatim to pcpu_map_pages().
 */
static int pcpu_alloc_pages(struct pcpu_chunk *chunk,
                            struct page **pages, int page_start, int page_end,
                            gfp_t gfp)
{
        unsigned int cpu, tcpu;
        int i;

        gfp |= __GFP_HIGHMEM;

        for_each_possible_cpu(cpu) {
                for (i = page_start; i < page_end; i++) {
                        struct page **pagep = &pages[pcpu_page_idx(cpu, i)];

                        *pagep = alloc_pages_node(cpu_to_node(cpu), gfp, 0);
                        if (!*pagep)
                                goto err;
                }
        }
        return 0;

err:
        while (--i >= page_start)
                __free_page(pages[pcpu_page_idx(cpu, i)]);

        for_each_possible_cpu(tcpu) {
                if (tcpu == cpu)
                        break;
                for (i = page_start; i < page_end; i++)
                        __free_page(pages[pcpu_page_idx(tcpu, i)]);
        }
        return -ENOMEM;
}

/**
 * pcpu_pre_unmap_flush - flush cache prior to unmapping
 * @chunk: chunk the regions to be flushed belongs to
 * @page_start: page index of the first page to be flushed
 * @page_end: page index of the last page to be flushed + 1
 *
 * Pages in [@page_start,@page_end) of @chunk are about to be
 * unmapped.  Flush cache.  As each flushing trial can be very
 * expensive, issue flush on the whole region at once rather than
 * doing it for each cpu.  This could be an overkill but is more
 * scalable.
 */
static void pcpu_pre_unmap_flush(struct pcpu_chunk *chunk,
                                 int page_start, int page_end)
{
        flush_cache_vunmap(
                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}

static void __pcpu_unmap_pages(unsigned long addr, int nr_pages)
{
        vunmap_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT));
}

/**
 * pcpu_unmap_pages - unmap pages out of a pcpu_chunk
 * @chunk: chunk of interest
 * @pages: pages array which can be used to pass information to free
 * @page_start: page index of the first page to unmap
 * @page_end: page index of the last page to unmap + 1
 *
 * For each cpu, unmap pages [@page_start,@page_end) out of @chunk.
 * Corresponding elements in @pages were cleared by the caller and can
 * be used to carry information to pcpu_free_pages() which will be
 * called after all unmaps are finished.  The caller should call
 * proper pre/post flush functions.
 */
static void pcpu_unmap_pages(struct pcpu_chunk *chunk,
                             struct page **pages, int page_start, int page_end)
{
        unsigned int cpu;
        int i;

        for_each_possible_cpu(cpu) {
                for (i = page_start; i < page_end; i++) {
                        struct page *page;

                        page = pcpu_chunk_page(chunk, cpu, i);
                        WARN_ON(!page);
                        pages[pcpu_page_idx(cpu, i)] = page;
                }
                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, cpu, page_start),
                                   page_end - page_start);
        }
}

/**
 * pcpu_post_unmap_tlb_flush - flush TLB after unmapping
 * @chunk: pcpu_chunk the regions to be flushed belong to
 * @page_start: page index of the first page to be flushed
 * @page_end: page index of the last page to be flushed + 1
 *
 * Pages [@page_start,@page_end) of @chunk have been unmapped.  Flush
 * TLB for the regions.  This can be skipped if the area is to be
 * returned to vmalloc as vmalloc will handle TLB flushing lazily.
 *
 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
 * for the whole region.
 */
static void pcpu_post_unmap_tlb_flush(struct pcpu_chunk *chunk,
                                      int page_start, int page_end)
{
        flush_tlb_kernel_range(
                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}

static int __pcpu_map_pages(unsigned long addr, struct page **pages,
                            int nr_pages)
{
        return vmap_pages_range_noflush(addr, addr + (nr_pages << PAGE_SHIFT),
                                        PAGE_KERNEL, pages, PAGE_SHIFT);
}

/**
 * pcpu_map_pages - map pages into a pcpu_chunk
 * @chunk: chunk of interest
 * @pages: pages array containing pages to be mapped
 * @page_start: page index of the first page to map
 * @page_end: page index of the last page to map + 1
 *
 * For each cpu, map pages [@page_start,@page_end) into @chunk.  The
 * caller is responsible for calling pcpu_post_map_flush() after all
 * mappings are complete.
 *
 * This function is responsible for setting up whatever is necessary for
 * reverse lookup (addr -> chunk).
 */
static int pcpu_map_pages(struct pcpu_chunk *chunk,
                          struct page **pages, int page_start, int page_end)
{
        unsigned int cpu, tcpu;
        int i, err;

        for_each_possible_cpu(cpu) {
                err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start),
                                       &pages[pcpu_page_idx(cpu, page_start)],
                                       page_end - page_start);
                if (err < 0)
                        goto err;

                for (i = page_start; i < page_end; i++)
                        pcpu_set_page_chunk(pages[pcpu_page_idx(cpu, i)],
                                            chunk);
        }
        return 0;
err:
        for_each_possible_cpu(tcpu) {
                __pcpu_unmap_pages(pcpu_chunk_addr(chunk, tcpu, page_start),
                                   page_end - page_start);
                if (tcpu == cpu)
                        break;
        }
        pcpu_post_unmap_tlb_flush(chunk, page_start, page_end);
        return err;
}

/**
 * pcpu_post_map_flush - flush cache after mapping
 * @chunk: pcpu_chunk the regions to be flushed belong to
 * @page_start: page index of the first page to be flushed
 * @page_end: page index of the last page to be flushed + 1
 *
 * Pages [@page_start,@page_end) of @chunk have been mapped.  Flush
 * cache.
 *
 * As with pcpu_pre_unmap_flush(), TLB flushing also is done at once
 * for the whole region.
 */
static void pcpu_post_map_flush(struct pcpu_chunk *chunk,
                                int page_start, int page_end)
{
        flush_cache_vmap(
                pcpu_chunk_addr(chunk, pcpu_low_unit_cpu, page_start),
                pcpu_chunk_addr(chunk, pcpu_high_unit_cpu, page_end));
}

/**
 * pcpu_populate_chunk - populate and map an area of a pcpu_chunk
 * @chunk: chunk of interest
 * @page_start: the start page
 * @page_end: the end page
 * @gfp: allocation flags passed to the underlying memory allocator
 *
 * For each cpu, populate and map pages [@page_start,@page_end) into
 * @chunk.
 *
 * CONTEXT:
 * pcpu_alloc_mutex, does GFP_KERNEL allocation.
 */
static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
                               int page_start, int page_end, gfp_t gfp)
{
        struct page **pages;

        pages = pcpu_get_pages();
        if (!pages)
                return -ENOMEM;

        if (pcpu_alloc_pages(chunk, pages, page_start, page_end, gfp))
                return -ENOMEM;

        if (pcpu_map_pages(chunk, pages, page_start, page_end)) {
                pcpu_free_pages(chunk, pages, page_start, page_end);
                return -ENOMEM;
        }
        pcpu_post_map_flush(chunk, page_start, page_end);

        return 0;
}

/**
 * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk
 * @chunk: chunk to depopulate
 * @page_start: the start page
 * @page_end: the end page
 *
 * For each cpu, depopulate and unmap pages [@page_start,@page_end)
 * from @chunk.
 *
 * Caller is required to call pcpu_post_unmap_tlb_flush() if not returning the
 * region back to vmalloc() which will lazily flush the tlb.
 *
 * CONTEXT:
 * pcpu_alloc_mutex.
 */
static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
                                  int page_start, int page_end)
{
        struct page **pages;

        /*
         * If control reaches here, there must have been at least one
         * successful population attempt so the temp pages array must
         * be available now.
         */
        pages = pcpu_get_pages();
        BUG_ON(!pages);

        /* unmap and free */
        pcpu_pre_unmap_flush(chunk, page_start, page_end);

        pcpu_unmap_pages(chunk, pages, page_start, page_end);

        pcpu_free_pages(chunk, pages, page_start, page_end);
}

static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp)
{
        struct pcpu_chunk *chunk;
        struct vm_struct **vms;

        chunk = pcpu_alloc_chunk(gfp);
        if (!chunk)
                return NULL;

        vms = pcpu_get_vm_areas(pcpu_group_offsets, pcpu_group_sizes,
                                pcpu_nr_groups, pcpu_atom_size);
        if (!vms) {
                pcpu_free_chunk(chunk);
                return NULL;
        }

        chunk->data = vms;
        chunk->base_addr = vms[0]->addr - pcpu_group_offsets[0];

        pcpu_stats_chunk_alloc();
        trace_percpu_create_chunk(chunk->base_addr);

        return chunk;
}

static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
{
        if (!chunk)
                return;

        pcpu_stats_chunk_dealloc();
        trace_percpu_destroy_chunk(chunk->base_addr);

        if (chunk->data)
                pcpu_free_vm_areas(chunk->data, pcpu_nr_groups);
        pcpu_free_chunk(chunk);
}

static struct page *pcpu_addr_to_page(void *addr)
{
        return vmalloc_to_page(addr);
}

static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
{
        /* no extra restriction */
        return 0;
}

/**
 * pcpu_should_reclaim_chunk - determine if a chunk should go into reclaim
 * @chunk: chunk of interest
 *
 * This is the entry point for percpu reclaim.  If a chunk qualifies, it is then
 * isolated and managed in separate lists at the back of pcpu_slot: sidelined
 * and to_depopulate respectively.  The to_depopulate list holds chunks slated
 * for depopulation.  They no longer contribute to pcpu_nr_empty_pop_pages once
 * they are on this list.  Once depopulated, they are moved onto the sidelined
 * list which enables them to be pulled back in for allocation if no other chunk
 * can suffice the allocation.
 */
static bool pcpu_should_reclaim_chunk(struct pcpu_chunk *chunk)
{
        /* do not reclaim either the first chunk or reserved chunk */
        if (chunk == pcpu_first_chunk || chunk == pcpu_reserved_chunk)
                return false;

        /*
         * If it is isolated, it may be on the sidelined list so move it back to
         * the to_depopulate list.  If we hit at least 1/4 pages empty pages AND
         * there is no system-wide shortage of empty pages aside from this
         * chunk, move it to the to_depopulate list.
         */
        return ((chunk->isolated && chunk->nr_empty_pop_pages) ||
                (pcpu_nr_empty_pop_pages >
                 (PCPU_EMPTY_POP_PAGES_HIGH + chunk->nr_empty_pop_pages) &&
                 chunk->nr_empty_pop_pages >= chunk->nr_pages / 4));
}
















































































   37 













   37 




   37 



   37 




















   37 





















































   39 
   38 
    2 
   37 






























































































































































































































   37 











   37 
   37 


   37 
   37 


















































   37 




































































































































































































































   37 


   37 

   37 
















   37 

   37 









   37 








































































































































   37 



















   37 










































   37 
























   37 



   37 








   37 





























   37 





























   37 










   37 





   37 





   37 


























   37 






   37 























   37 





















   37 



   37 





















   37 






















   37 









   37 




































































































   37 









   37 










   37 




   37 




























































































































































































































































































































































































   37 



















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   37 




















   37 




   37 
   36 
    6 












   36 



























































   37 































   37 
   37 





   37 

















   37 








































































































   37 






















   37 


   37 


   37 

   37 




   37 



   37 









































































































    2 























   41 






   38 



    2 












    4 




    2 
    4 









    4 



























































































































































































































   37 




























   37 

   37 

   37 


































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   37 

   37 
   37 
   37 
































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/signal.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  1997-11-02  Modified for POSIX.1b signals by Richard Henderson
 *
 *  2003-06-02  Jim Houston - Concurrent Computer Corp.
 *                Changes to use preallocated sigqueue structures
 *                to allow signals to be sent reliably.
 */

#include <linux/slab.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/user.h>
#include <linux/sched/debug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/tty.h>
#include <linux/binfmts.h>
#include <linux/coredump.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/signalfd.h>
#include <linux/ratelimit.h>
#include <linux/task_work.h>
#include <linux/capability.h>
#include <linux/freezer.h>
#include <linux/pid_namespace.h>
#include <linux/nsproxy.h>
#include <linux/user_namespace.h>
#include <linux/uprobes.h>
#include <linux/compat.h>
#include <linux/cn_proc.h>
#include <linux/compiler.h>
#include <linux/posix-timers.h>
#include <linux/cgroup.h>
#include <linux/audit.h>
#include <linux/sysctl.h>
#include <uapi/linux/pidfd.h>

#define CREATE_TRACE_POINTS
#include <trace/events/signal.h>

#include <asm/param.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
#include <asm/siginfo.h>
#include <asm/cacheflush.h>
#include <asm/syscall.h>        /* for syscall_get_* */

#include "time/posix-timers.h"

/*
 * SLAB caches for signal bits.
 */

static struct kmem_cache *sigqueue_cachep;

int print_fatal_signals __read_mostly;

static void __user *sig_handler(struct task_struct *t, int sig)
{
        return t->sighand->action[sig - 1].sa.sa_handler;
}

static inline bool sig_handler_ignored(void __user *handler, int sig)
{
        /* Is it explicitly or implicitly ignored? */
        return handler == SIG_IGN ||
               (handler == SIG_DFL && sig_kernel_ignore(sig));
}

static bool sig_task_ignored(struct task_struct *t, int sig, bool force)
{
        void __user *handler;

        handler = sig_handler(t, sig);

        /* SIGKILL and SIGSTOP may not be sent to the global init */
        if (unlikely(is_global_init(t) && sig_kernel_only(sig)))
                return true;

        if (unlikely(t->signal->flags & SIGNAL_UNKILLABLE) &&
            handler == SIG_DFL && !(force && sig_kernel_only(sig)))
                return true;

        /* Only allow kernel generated signals to this kthread */
        if (unlikely((t->flags & PF_KTHREAD) &&
                     (handler == SIG_KTHREAD_KERNEL) && !force))
                return true;

        return sig_handler_ignored(handler, sig);
}

static bool sig_ignored(struct task_struct *t, int sig, bool force)
{
        /*
         * Blocked signals are never ignored, since the
         * signal handler may change by the time it is
         * unblocked.
         */
        if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig))
                return false;

        /*
         * Tracers may want to know about even ignored signal unless it
         * is SIGKILL which can't be reported anyway but can be ignored
         * by SIGNAL_UNKILLABLE task.
         */
        if (t->ptrace && sig != SIGKILL)
                return false;

        return sig_task_ignored(t, sig, force);
}

/*
 * Re-calculate pending state from the set of locally pending
 * signals, globally pending signals, and blocked signals.
 */
static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked)
{
        unsigned long ready;
        long i;

        switch (_NSIG_WORDS) {
        default:
                for (i = _NSIG_WORDS, ready = 0; --i >= 0 ;)
                        ready |= signal->sig[i] &~ blocked->sig[i];
                break;

        case 4: ready  = signal->sig[3] &~ blocked->sig[3];
                ready |= signal->sig[2] &~ blocked->sig[2];
                ready |= signal->sig[1] &~ blocked->sig[1];
                ready |= signal->sig[0] &~ blocked->sig[0];
                break;

        case 2: ready  = signal->sig[1] &~ blocked->sig[1];
                ready |= signal->sig[0] &~ blocked->sig[0];
                break;

        case 1: ready  = signal->sig[0] &~ blocked->sig[0];
        }
        return ready !=        0;
}

#define PENDING(p,b) has_pending_signals(&(p)->signal, (b))

static bool recalc_sigpending_tsk(struct task_struct *t)
{
        if ((t->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) ||
            PENDING(&t->pending, &t->blocked) ||
            PENDING(&t->signal->shared_pending, &t->blocked) ||
            cgroup_task_frozen(t)) {
                set_tsk_thread_flag(t, TIF_SIGPENDING);
                return true;
        }

        /*
         * We must never clear the flag in another thread, or in current
         * when it's possible the current syscall is returning -ERESTART*.
         * So we don't clear it here, and only callers who know they should do.
         */
        return false;
}

void recalc_sigpending(void)
{
        if (!recalc_sigpending_tsk(current) && !freezing(current)) {
                if (unlikely(test_thread_flag(TIF_SIGPENDING)))
                        clear_thread_flag(TIF_SIGPENDING);
        }
}
EXPORT_SYMBOL(recalc_sigpending);

void calculate_sigpending(void)
{
        /* Have any signals or users of TIF_SIGPENDING been delayed
         * until after fork?
         */
        spin_lock_irq(&current->sighand->siglock);
        set_tsk_thread_flag(current, TIF_SIGPENDING);
        recalc_sigpending();
        spin_unlock_irq(&current->sighand->siglock);
}

/* Given the mask, find the first available signal that should be serviced. */

#define SYNCHRONOUS_MASK \
        (sigmask(SIGSEGV) | sigmask(SIGBUS) | sigmask(SIGILL) | \
         sigmask(SIGTRAP) | sigmask(SIGFPE) | sigmask(SIGSYS))

int next_signal(struct sigpending *pending, sigset_t *mask)
{
        unsigned long i, *s, *m, x;
        int sig = 0;

        s = pending->signal.sig;
        m = mask->sig;

        /*
         * Handle the first word specially: it contains the
         * synchronous signals that need to be dequeued first.
         */
        x = *s &~ *m;
        if (x) {
                if (x & SYNCHRONOUS_MASK)
                        x &= SYNCHRONOUS_MASK;
                sig = ffz(~x) + 1;
                return sig;
        }

        switch (_NSIG_WORDS) {
        default:
                for (i = 1; i < _NSIG_WORDS; ++i) {
                        x = *++s &~ *++m;
                        if (!x)
                                continue;
                        sig = ffz(~x) + i*_NSIG_BPW + 1;
                        break;
                }
                break;

        case 2:
                x = s[1] &~ m[1];
                if (!x)
                        break;
                sig = ffz(~x) + _NSIG_BPW + 1;
                break;

        case 1:
                /* Nothing to do */
                break;
        }

        return sig;
}

static inline void print_dropped_signal(int sig)
{
        static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);

        if (!print_fatal_signals)
                return;

        if (!__ratelimit(&ratelimit_state))
                return;

        pr_info("%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n",
                                current->comm, current->pid, sig);
}

/**
 * task_set_jobctl_pending - set jobctl pending bits
 * @task: target task
 * @mask: pending bits to set
 *
 * Clear @mask from @task->jobctl.  @mask must be subset of
 * %JOBCTL_PENDING_MASK | %JOBCTL_STOP_CONSUME | %JOBCTL_STOP_SIGMASK |
 * %JOBCTL_TRAPPING.  If stop signo is being set, the existing signo is
 * cleared.  If @task is already being killed or exiting, this function
 * becomes noop.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 *
 * RETURNS:
 * %true if @mask is set, %false if made noop because @task was dying.
 */
bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask)
{
        BUG_ON(mask & ~(JOBCTL_PENDING_MASK | JOBCTL_STOP_CONSUME |
                        JOBCTL_STOP_SIGMASK | JOBCTL_TRAPPING));
        BUG_ON((mask & JOBCTL_TRAPPING) && !(mask & JOBCTL_PENDING_MASK));

        if (unlikely(fatal_signal_pending(task) || (task->flags & PF_EXITING)))
                return false;

        if (mask & JOBCTL_STOP_SIGMASK)
                task->jobctl &= ~JOBCTL_STOP_SIGMASK;

        task->jobctl |= mask;
        return true;
}

/**
 * task_clear_jobctl_trapping - clear jobctl trapping bit
 * @task: target task
 *
 * If JOBCTL_TRAPPING is set, a ptracer is waiting for us to enter TRACED.
 * Clear it and wake up the ptracer.  Note that we don't need any further
 * locking.  @task->siglock guarantees that @task->parent points to the
 * ptracer.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
void task_clear_jobctl_trapping(struct task_struct *task)
{
        if (unlikely(task->jobctl & JOBCTL_TRAPPING)) {
                task->jobctl &= ~JOBCTL_TRAPPING;
                smp_mb();        /* advised by wake_up_bit() */
                wake_up_bit(&task->jobctl, JOBCTL_TRAPPING_BIT);
        }
}

/**
 * task_clear_jobctl_pending - clear jobctl pending bits
 * @task: target task
 * @mask: pending bits to clear
 *
 * Clear @mask from @task->jobctl.  @mask must be subset of
 * %JOBCTL_PENDING_MASK.  If %JOBCTL_STOP_PENDING is being cleared, other
 * STOP bits are cleared together.
 *
 * If clearing of @mask leaves no stop or trap pending, this function calls
 * task_clear_jobctl_trapping().
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
void task_clear_jobctl_pending(struct task_struct *task, unsigned long mask)
{
        BUG_ON(mask & ~JOBCTL_PENDING_MASK);

        if (mask & JOBCTL_STOP_PENDING)
                mask |= JOBCTL_STOP_CONSUME | JOBCTL_STOP_DEQUEUED;

        task->jobctl &= ~mask;

        if (!(task->jobctl & JOBCTL_PENDING_MASK))
                task_clear_jobctl_trapping(task);
}

/**
 * task_participate_group_stop - participate in a group stop
 * @task: task participating in a group stop
 *
 * @task has %JOBCTL_STOP_PENDING set and is participating in a group stop.
 * Group stop states are cleared and the group stop count is consumed if
 * %JOBCTL_STOP_CONSUME was set.  If the consumption completes the group
 * stop, the appropriate `SIGNAL_*` flags are set.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 *
 * RETURNS:
 * %true if group stop completion should be notified to the parent, %false
 * otherwise.
 */
static bool task_participate_group_stop(struct task_struct *task)
{
        struct signal_struct *sig = task->signal;
        bool consume = task->jobctl & JOBCTL_STOP_CONSUME;

        WARN_ON_ONCE(!(task->jobctl & JOBCTL_STOP_PENDING));

        task_clear_jobctl_pending(task, JOBCTL_STOP_PENDING);

        if (!consume)
                return false;

        if (!WARN_ON_ONCE(sig->group_stop_count == 0))
                sig->group_stop_count--;

        /*
         * Tell the caller to notify completion iff we are entering into a
         * fresh group stop.  Read comment in do_signal_stop() for details.
         */
        if (!sig->group_stop_count && !(sig->flags & SIGNAL_STOP_STOPPED)) {
                signal_set_stop_flags(sig, SIGNAL_STOP_STOPPED);
                return true;
        }
        return false;
}

void task_join_group_stop(struct task_struct *task)
{
        unsigned long mask = current->jobctl & JOBCTL_STOP_SIGMASK;
        struct signal_struct *sig = current->signal;

        if (sig->group_stop_count) {
                sig->group_stop_count++;
                mask |= JOBCTL_STOP_CONSUME;
        } else if (!(sig->flags & SIGNAL_STOP_STOPPED))
                return;

        /* Have the new thread join an on-going signal group stop */
        task_set_jobctl_pending(task, mask | JOBCTL_STOP_PENDING);
}

static struct ucounts *sig_get_ucounts(struct task_struct *t, int sig,
                                       int override_rlimit)
{
        struct ucounts *ucounts;
        long sigpending;

        /*
         * Protect access to @t credentials. This can go away when all
         * callers hold rcu read lock.
         *
         * NOTE! A pending signal will hold on to the user refcount,
         * and we get/put the refcount only when the sigpending count
         * changes from/to zero.
         */
        rcu_read_lock();
        ucounts = task_ucounts(t);
        sigpending = inc_rlimit_get_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING,
                                            override_rlimit);
        rcu_read_unlock();
        if (!sigpending)
                return NULL;

        if (unlikely(!override_rlimit && sigpending > task_rlimit(t, RLIMIT_SIGPENDING))) {
                dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
                print_dropped_signal(sig);
                return NULL;
        }

        return ucounts;
}

static void __sigqueue_init(struct sigqueue *q, struct ucounts *ucounts,
                            const unsigned int sigqueue_flags)
{
        INIT_LIST_HEAD(&q->list);
        q->flags = sigqueue_flags;
        q->ucounts = ucounts;
}

/*
 * allocate a new signal queue record
 * - this may be called without locks if and only if t == current, otherwise an
 *   appropriate lock must be held to stop the target task from exiting
 */
static struct sigqueue *sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
                                       int override_rlimit)
{
        struct ucounts *ucounts = sig_get_ucounts(t, sig, override_rlimit);
        struct sigqueue *q;

        if (!ucounts)
                return NULL;

        q = kmem_cache_alloc(sigqueue_cachep, gfp_flags);
        if (!q) {
                dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
                return NULL;
        }

        __sigqueue_init(q, ucounts, 0);
        return q;
}

static void __sigqueue_free(struct sigqueue *q)
{
        if (q->flags & SIGQUEUE_PREALLOC) {
                posixtimer_sigqueue_putref(q);
                return;
        }
        if (q->ucounts) {
                dec_rlimit_put_ucounts(q->ucounts, UCOUNT_RLIMIT_SIGPENDING);
                q->ucounts = NULL;
        }
        kmem_cache_free(sigqueue_cachep, q);
}

void flush_sigqueue(struct sigpending *queue)
{
        struct sigqueue *q;

        sigemptyset(&queue->signal);
        while (!list_empty(&queue->list)) {
                q = list_entry(queue->list.next, struct sigqueue , list);
                list_del_init(&q->list);
                __sigqueue_free(q);
        }
}

/*
 * Flush all pending signals for this kthread.
 */
void flush_signals(struct task_struct *t)
{
        unsigned long flags;

        spin_lock_irqsave(&t->sighand->siglock, flags);
        clear_tsk_thread_flag(t, TIF_SIGPENDING);
        flush_sigqueue(&t->pending);
        flush_sigqueue(&t->signal->shared_pending);
        spin_unlock_irqrestore(&t->sighand->siglock, flags);
}
EXPORT_SYMBOL(flush_signals);

void ignore_signals(struct task_struct *t)
{
        int i;

        for (i = 0; i < _NSIG; ++i)
                t->sighand->action[i].sa.sa_handler = SIG_IGN;

        flush_signals(t);
}

/*
 * Flush all handlers for a task.
 */

void
flush_signal_handlers(struct task_struct *t, int force_default)
{
        int i;
        struct k_sigaction *ka = &t->sighand->action[0];
        for (i = _NSIG ; i != 0 ; i--) {
                if (force_default || ka->sa.sa_handler != SIG_IGN)
                        ka->sa.sa_handler = SIG_DFL;
                ka->sa.sa_flags = 0;
#ifdef __ARCH_HAS_SA_RESTORER
                ka->sa.sa_restorer = NULL;
#endif
                sigemptyset(&ka->sa.sa_mask);
                ka++;
        }
}

bool unhandled_signal(struct task_struct *tsk, int sig)
{
        void __user *handler = tsk->sighand->action[sig-1].sa.sa_handler;
        if (is_global_init(tsk))
                return true;

        if (handler != SIG_IGN && handler != SIG_DFL)
                return false;

        /* If dying, we handle all new signals by ignoring them */
        if (fatal_signal_pending(tsk))
                return false;

        /* if ptraced, let the tracer determine */
        return !tsk->ptrace;
}

static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *info,
                           struct sigqueue **timer_sigq)
{
        struct sigqueue *q, *first = NULL;

        /*
         * Collect the siginfo appropriate to this signal.  Check if
         * there is another siginfo for the same signal.
        */
        list_for_each_entry(q, &list->list, list) {
                if (q->info.si_signo == sig) {
                        if (first)
                                goto still_pending;
                        first = q;
                }
        }

        sigdelset(&list->signal, sig);

        if (first) {
still_pending:
                list_del_init(&first->list);
                copy_siginfo(info, &first->info);

                /*
                 * posix-timer signals are preallocated and freed when the last
                 * reference count is dropped in posixtimer_deliver_signal() or
                 * immediately on timer deletion when the signal is not pending.
                 * Spare the extra round through __sigqueue_free() which is
                 * ignoring preallocated signals.
                 */
                if (unlikely((first->flags & SIGQUEUE_PREALLOC) && (info->si_code == SI_TIMER)))
                        *timer_sigq = first;
                else
                        __sigqueue_free(first);
        } else {
                /*
                 * Ok, it wasn't in the queue.  This must be
                 * a fast-pathed signal or we must have been
                 * out of queue space.  So zero out the info.
                 */
                clear_siginfo(info);
                info->si_signo = sig;
                info->si_errno = 0;
                info->si_code = SI_USER;
                info->si_pid = 0;
                info->si_uid = 0;
        }
}

static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
                            kernel_siginfo_t *info, struct sigqueue **timer_sigq)
{
        int sig = next_signal(pending, mask);

        if (sig)
                collect_signal(sig, pending, info, timer_sigq);
        return sig;
}

/*
 * Try to dequeue a signal. If a deliverable signal is found fill in the
 * caller provided siginfo and return the signal number. Otherwise return
 * 0.
 */
int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type)
{
        struct task_struct *tsk = current;
        struct sigqueue *timer_sigq;
        int signr;

        lockdep_assert_held(&tsk->sighand->siglock);

again:
        *type = PIDTYPE_PID;
        timer_sigq = NULL;
        signr = __dequeue_signal(&tsk->pending, mask, info, &timer_sigq);
        if (!signr) {
                *type = PIDTYPE_TGID;
                signr = __dequeue_signal(&tsk->signal->shared_pending,
                                         mask, info, &timer_sigq);

                if (unlikely(signr == SIGALRM))
                        posixtimer_rearm_itimer(tsk);
        }

        recalc_sigpending();
        if (!signr)
                return 0;

        if (unlikely(sig_kernel_stop(signr))) {
                /*
                 * Set a marker that we have dequeued a stop signal.  Our
                 * caller might release the siglock and then the pending
                 * stop signal it is about to process is no longer in the
                 * pending bitmasks, but must still be cleared by a SIGCONT
                 * (and overruled by a SIGKILL).  So those cases clear this
                 * shared flag after we've set it.  Note that this flag may
                 * remain set after the signal we return is ignored or
                 * handled.  That doesn't matter because its only purpose
                 * is to alert stop-signal processing code when another
                 * processor has come along and cleared the flag.
                 */
                current->jobctl |= JOBCTL_STOP_DEQUEUED;
        }

        if (IS_ENABLED(CONFIG_POSIX_TIMERS) && unlikely(timer_sigq)) {
                if (!posixtimer_deliver_signal(info, timer_sigq))
                        goto again;
        }

        return signr;
}
EXPORT_SYMBOL_GPL(dequeue_signal);

static int dequeue_synchronous_signal(kernel_siginfo_t *info)
{
        struct task_struct *tsk = current;
        struct sigpending *pending = &tsk->pending;
        struct sigqueue *q, *sync = NULL;

        /*
         * Might a synchronous signal be in the queue?
         */
        if (!((pending->signal.sig[0] & ~tsk->blocked.sig[0]) & SYNCHRONOUS_MASK))
                return 0;

        /*
         * Return the first synchronous signal in the queue.
         */
        list_for_each_entry(q, &pending->list, list) {
                /* Synchronous signals have a positive si_code */
                if ((q->info.si_code > SI_USER) &&
                    (sigmask(q->info.si_signo) & SYNCHRONOUS_MASK)) {
                        sync = q;
                        goto next;
                }
        }
        return 0;
next:
        /*
         * Check if there is another siginfo for the same signal.
         */
        list_for_each_entry_continue(q, &pending->list, list) {
                if (q->info.si_signo == sync->info.si_signo)
                        goto still_pending;
        }

        sigdelset(&pending->signal, sync->info.si_signo);
        recalc_sigpending();
still_pending:
        list_del_init(&sync->list);
        copy_siginfo(info, &sync->info);
        __sigqueue_free(sync);
        return info->si_signo;
}

/*
 * Tell a process that it has a new active signal..
 *
 * NOTE! we rely on the previous spin_lock to
 * lock interrupts for us! We can only be called with
 * "siglock" held, and the local interrupt must
 * have been disabled when that got acquired!
 *
 * No need to set need_resched since signal event passing
 * goes through ->blocked
 */
void signal_wake_up_state(struct task_struct *t, unsigned int state)
{
        lockdep_assert_held(&t->sighand->siglock);

        set_tsk_thread_flag(t, TIF_SIGPENDING);

        /*
         * TASK_WAKEKILL also means wake it up in the stopped/traced/killable
         * case. We don't check t->state here because there is a race with it
         * executing another processor and just now entering stopped state.
         * By using wake_up_state, we ensure the process will wake up and
         * handle its death signal.
         */
        if (!wake_up_state(t, state | TASK_INTERRUPTIBLE))
                kick_process(t);
}

static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q);

static void sigqueue_free_ignored(struct task_struct *tsk, struct sigqueue *q)
{
        if (likely(!(q->flags & SIGQUEUE_PREALLOC) || q->info.si_code != SI_TIMER))
                __sigqueue_free(q);
        else
                posixtimer_sig_ignore(tsk, q);
}

/* Remove signals in mask from the pending set and queue. */
static void flush_sigqueue_mask(struct task_struct *p, sigset_t *mask, struct sigpending *s)
{
        struct sigqueue *q, *n;
        sigset_t m;

        lockdep_assert_held(&p->sighand->siglock);

        sigandsets(&m, mask, &s->signal);
        if (sigisemptyset(&m))
                return;

        sigandnsets(&s->signal, &s->signal, mask);
        list_for_each_entry_safe(q, n, &s->list, list) {
                if (sigismember(mask, q->info.si_signo)) {
                        list_del_init(&q->list);
                        sigqueue_free_ignored(p, q);
                }
        }
}

static inline int is_si_special(const struct kernel_siginfo *info)
{
        return info <= SEND_SIG_PRIV;
}

static inline bool si_fromuser(const struct kernel_siginfo *info)
{
        return info == SEND_SIG_NOINFO ||
                (!is_si_special(info) && SI_FROMUSER(info));
}

/*
 * called with RCU read lock from check_kill_permission()
 */
static bool kill_ok_by_cred(struct task_struct *t)
{
        const struct cred *cred = current_cred();
        const struct cred *tcred = __task_cred(t);

        return uid_eq(cred->euid, tcred->suid) ||
               uid_eq(cred->euid, tcred->uid) ||
               uid_eq(cred->uid, tcred->suid) ||
               uid_eq(cred->uid, tcred->uid) ||
               ns_capable(tcred->user_ns, CAP_KILL);
}

/*
 * Bad permissions for sending the signal
 * - the caller must hold the RCU read lock
 */
static int check_kill_permission(int sig, struct kernel_siginfo *info,
                                 struct task_struct *t)
{
        struct pid *sid;
        int error;

        if (!valid_signal(sig))
                return -EINVAL;

        if (!si_fromuser(info))
                return 0;

        error = audit_signal_info(sig, t); /* Let audit system see the signal */
        if (error)
                return error;

        if (!same_thread_group(current, t) &&
            !kill_ok_by_cred(t)) {
                switch (sig) {
                case SIGCONT:
                        sid = task_session(t);
                        /*
                         * We don't return the error if sid == NULL. The
                         * task was unhashed, the caller must notice this.
                         */
                        if (!sid || sid == task_session(current))
                                break;
                        fallthrough;
                default:
                        return -EPERM;
                }
        }

        return security_task_kill(t, info, sig, NULL);
}

/**
 * ptrace_trap_notify - schedule trap to notify ptracer
 * @t: tracee wanting to notify tracer
 *
 * This function schedules sticky ptrace trap which is cleared on the next
 * TRAP_STOP to notify ptracer of an event.  @t must have been seized by
 * ptracer.
 *
 * If @t is running, STOP trap will be taken.  If trapped for STOP and
 * ptracer is listening for events, tracee is woken up so that it can
 * re-trap for the new event.  If trapped otherwise, STOP trap will be
 * eventually taken without returning to userland after the existing traps
 * are finished by PTRACE_CONT.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
static void ptrace_trap_notify(struct task_struct *t)
{
        WARN_ON_ONCE(!(t->ptrace & PT_SEIZED));
        lockdep_assert_held(&t->sighand->siglock);

        task_set_jobctl_pending(t, JOBCTL_TRAP_NOTIFY);
        ptrace_signal_wake_up(t, t->jobctl & JOBCTL_LISTENING);
}

/*
 * Handle magic process-wide effects of stop/continue signals. Unlike
 * the signal actions, these happen immediately at signal-generation
 * time regardless of blocking, ignoring, or handling.  This does the
 * actual continuing for SIGCONT, but not the actual stopping for stop
 * signals. The process stop is done as a signal action for SIG_DFL.
 *
 * Returns true if the signal should be actually delivered, otherwise
 * it should be dropped.
 */
static bool prepare_signal(int sig, struct task_struct *p, bool force)
{
        struct signal_struct *signal = p->signal;
        struct task_struct *t;
        sigset_t flush;

        if (signal->flags & SIGNAL_GROUP_EXIT) {
                if (signal->core_state)
                        return sig == SIGKILL;
                /*
                 * The process is in the middle of dying, drop the signal.
                 */
                return false;
        } else if (sig_kernel_stop(sig)) {
                /*
                 * This is a stop signal.  Remove SIGCONT from all queues.
                 */
                siginitset(&flush, sigmask(SIGCONT));
                flush_sigqueue_mask(p, &flush, &signal->shared_pending);
                for_each_thread(p, t)
                        flush_sigqueue_mask(p, &flush, &t->pending);
        } else if (sig == SIGCONT) {
                unsigned int why;
                /*
                 * Remove all stop signals from all queues, wake all threads.
                 */
                siginitset(&flush, SIG_KERNEL_STOP_MASK);
                flush_sigqueue_mask(p, &flush, &signal->shared_pending);
                for_each_thread(p, t) {
                        flush_sigqueue_mask(p, &flush, &t->pending);
                        task_clear_jobctl_pending(t, JOBCTL_STOP_PENDING);
                        if (likely(!(t->ptrace & PT_SEIZED))) {
                                t->jobctl &= ~JOBCTL_STOPPED;
                                wake_up_state(t, __TASK_STOPPED);
                        } else
                                ptrace_trap_notify(t);
                }

                /*
                 * Notify the parent with CLD_CONTINUED if we were stopped.
                 *
                 * If we were in the middle of a group stop, we pretend it
                 * was already finished, and then continued. Since SIGCHLD
                 * doesn't queue we report only CLD_STOPPED, as if the next
                 * CLD_CONTINUED was dropped.
                 */
                why = 0;
                if (signal->flags & SIGNAL_STOP_STOPPED)
                        why |= SIGNAL_CLD_CONTINUED;
                else if (signal->group_stop_count)
                        why |= SIGNAL_CLD_STOPPED;

                if (why) {
                        /*
                         * The first thread which returns from do_signal_stop()
                         * will take ->siglock, notice SIGNAL_CLD_MASK, and
                         * notify its parent. See get_signal().
                         */
                        signal_set_stop_flags(signal, why | SIGNAL_STOP_CONTINUED);
                        signal->group_stop_count = 0;
                        signal->group_exit_code = 0;
                }
        }

        return !sig_ignored(p, sig, force);
}

/*
 * Test if P wants to take SIG.  After we've checked all threads with this,
 * it's equivalent to finding no threads not blocking SIG.  Any threads not
 * blocking SIG were ruled out because they are not running and already
 * have pending signals.  Such threads will dequeue from the shared queue
 * as soon as they're available, so putting the signal on the shared queue
 * will be equivalent to sending it to one such thread.
 */
static inline bool wants_signal(int sig, struct task_struct *p)
{
        if (sigismember(&p->blocked, sig))
                return false;

        if (p->flags & PF_EXITING)
                return false;

        if (sig == SIGKILL)
                return true;

        if (task_is_stopped_or_traced(p))
                return false;

        return task_curr(p) || !task_sigpending(p);
}

static void complete_signal(int sig, struct task_struct *p, enum pid_type type)
{
        struct signal_struct *signal = p->signal;
        struct task_struct *t;

        /*
         * Now find a thread we can wake up to take the signal off the queue.
         *
         * Try the suggested task first (may or may not be the main thread).
         */
        if (wants_signal(sig, p))
                t = p;
        else if ((type == PIDTYPE_PID) || thread_group_empty(p))
                /*
                 * There is just one thread and it does not need to be woken.
                 * It will dequeue unblocked signals before it runs again.
                 */
                return;
        else {
                /*
                 * Otherwise try to find a suitable thread.
                 */
                t = signal->curr_target;
                while (!wants_signal(sig, t)) {
                        t = next_thread(t);
                        if (t == signal->curr_target)
                                /*
                                 * No thread needs to be woken.
                                 * Any eligible threads will see
                                 * the signal in the queue soon.
                                 */
                                return;
                }
                signal->curr_target = t;
        }

        /*
         * Found a killable thread.  If the signal will be fatal,
         * then start taking the whole group down immediately.
         */
        if (sig_fatal(p, sig) &&
            (signal->core_state || !(signal->flags & SIGNAL_GROUP_EXIT)) &&
            !sigismember(&t->real_blocked, sig) &&
            (sig == SIGKILL || !p->ptrace)) {
                /*
                 * This signal will be fatal to the whole group.
                 */
                if (!sig_kernel_coredump(sig)) {
                        /*
                         * Start a group exit and wake everybody up.
                         * This way we don't have other threads
                         * running and doing things after a slower
                         * thread has the fatal signal pending.
                         */
                        signal->flags = SIGNAL_GROUP_EXIT;
                        signal->group_exit_code = sig;
                        signal->group_stop_count = 0;
                        __for_each_thread(signal, t) {
                                task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
                                sigaddset(&t->pending.signal, SIGKILL);
                                signal_wake_up(t, 1);
                        }
                        return;
                }
        }

        /*
         * The signal is already in the shared-pending queue.
         * Tell the chosen thread to wake up and dequeue it.
         */
        signal_wake_up(t, sig == SIGKILL);
        return;
}

static inline bool legacy_queue(struct sigpending *signals, int sig)
{
        return (sig < SIGRTMIN) && sigismember(&signals->signal, sig);
}

static int __send_signal_locked(int sig, struct kernel_siginfo *info,
                                struct task_struct *t, enum pid_type type, bool force)
{
        struct sigpending *pending;
        struct sigqueue *q;
        int override_rlimit;
        int ret = 0, result;

        lockdep_assert_held(&t->sighand->siglock);

        result = TRACE_SIGNAL_IGNORED;
        if (!prepare_signal(sig, t, force))
                goto ret;

        pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
        /*
         * Short-circuit ignored signals and support queuing
         * exactly one non-rt signal, so that we can get more
         * detailed information about the cause of the signal.
         */
        result = TRACE_SIGNAL_ALREADY_PENDING;
        if (legacy_queue(pending, sig))
                goto ret;

        result = TRACE_SIGNAL_DELIVERED;
        /*
         * Skip useless siginfo allocation for SIGKILL and kernel threads.
         */
        if ((sig == SIGKILL) || (t->flags & PF_KTHREAD))
                goto out_set;

        /*
         * Real-time signals must be queued if sent by sigqueue, or
         * some other real-time mechanism.  It is implementation
         * defined whether kill() does so.  We attempt to do so, on
         * the principle of least surprise, but since kill is not
         * allowed to fail with EAGAIN when low on memory we just
         * make sure at least one signal gets delivered and don't
         * pass on the info struct.
         */
        if (sig < SIGRTMIN)
                override_rlimit = (is_si_special(info) || info->si_code >= 0);
        else
                override_rlimit = 0;

        q = sigqueue_alloc(sig, t, GFP_ATOMIC, override_rlimit);

        if (q) {
                list_add_tail(&q->list, &pending->list);
                switch ((unsigned long) info) {
                case (unsigned long) SEND_SIG_NOINFO:
                        clear_siginfo(&q->info);
                        q->info.si_signo = sig;
                        q->info.si_errno = 0;
                        q->info.si_code = SI_USER;
                        q->info.si_pid = task_tgid_nr_ns(current,
                                                        task_active_pid_ns(t));
                        rcu_read_lock();
                        q->info.si_uid =
                                from_kuid_munged(task_cred_xxx(t, user_ns),
                                                 current_uid());
                        rcu_read_unlock();
                        break;
                case (unsigned long) SEND_SIG_PRIV:
                        clear_siginfo(&q->info);
                        q->info.si_signo = sig;
                        q->info.si_errno = 0;
                        q->info.si_code = SI_KERNEL;
                        q->info.si_pid = 0;
                        q->info.si_uid = 0;
                        break;
                default:
                        copy_siginfo(&q->info, info);
                        break;
                }
        } else if (!is_si_special(info) &&
                   sig >= SIGRTMIN && info->si_code != SI_USER) {
                /*
                 * Queue overflow, abort.  We may abort if the
                 * signal was rt and sent by user using something
                 * other than kill().
                 */
                result = TRACE_SIGNAL_OVERFLOW_FAIL;
                ret = -EAGAIN;
                goto ret;
        } else {
                /*
                 * This is a silent loss of information.  We still
                 * send the signal, but the *info bits are lost.
                 */
                result = TRACE_SIGNAL_LOSE_INFO;
        }

out_set:
        signalfd_notify(t, sig);
        sigaddset(&pending->signal, sig);

        /* Let multiprocess signals appear after on-going forks */
        if (type > PIDTYPE_TGID) {
                struct multiprocess_signals *delayed;
                hlist_for_each_entry(delayed, &t->signal->multiprocess, node) {
                        sigset_t *signal = &delayed->signal;
                        /* Can't queue both a stop and a continue signal */
                        if (sig == SIGCONT)
                                sigdelsetmask(signal, SIG_KERNEL_STOP_MASK);
                        else if (sig_kernel_stop(sig))
                                sigdelset(signal, SIGCONT);
                        sigaddset(signal, sig);
                }
        }

        complete_signal(sig, t, type);
ret:
        trace_signal_generate(sig, info, t, type != PIDTYPE_PID, result);
        return ret;
}

static inline bool has_si_pid_and_uid(struct kernel_siginfo *info)
{
        bool ret = false;
        switch (siginfo_layout(info->si_signo, info->si_code)) {
        case SIL_KILL:
        case SIL_CHLD:
        case SIL_RT:
                ret = true;
                break;
        case SIL_TIMER:
        case SIL_POLL:
        case SIL_FAULT:
        case SIL_FAULT_TRAPNO:
        case SIL_FAULT_MCEERR:
        case SIL_FAULT_BNDERR:
        case SIL_FAULT_PKUERR:
        case SIL_FAULT_PERF_EVENT:
        case SIL_SYS:
                ret = false;
                break;
        }
        return ret;
}

int send_signal_locked(int sig, struct kernel_siginfo *info,
                       struct task_struct *t, enum pid_type type)
{
        /* Should SIGKILL or SIGSTOP be received by a pid namespace init? */
        bool force = false;

        if (info == SEND_SIG_NOINFO) {
                /* Force if sent from an ancestor pid namespace */
                force = !task_pid_nr_ns(current, task_active_pid_ns(t));
        } else if (info == SEND_SIG_PRIV) {
                /* Don't ignore kernel generated signals */
                force = true;
        } else if (has_si_pid_and_uid(info)) {
                /* SIGKILL and SIGSTOP is special or has ids */
                struct user_namespace *t_user_ns;

                rcu_read_lock();
                t_user_ns = task_cred_xxx(t, user_ns);
                if (current_user_ns() != t_user_ns) {
                        kuid_t uid = make_kuid(current_user_ns(), info->si_uid);
                        info->si_uid = from_kuid_munged(t_user_ns, uid);
                }
                rcu_read_unlock();

                /* A kernel generated signal? */
                force = (info->si_code == SI_KERNEL);

                /* From an ancestor pid namespace? */
                if (!task_pid_nr_ns(current, task_active_pid_ns(t))) {
                        info->si_pid = 0;
                        force = true;
                }
        }
        return __send_signal_locked(sig, info, t, type, force);
}

static void print_fatal_signal(int signr)
{
        struct pt_regs *regs = task_pt_regs(current);
        struct file *exe_file;

        exe_file = get_task_exe_file(current);
        if (exe_file) {
                pr_info("%pD: %s: potentially unexpected fatal signal %d.\n",
                        exe_file, current->comm, signr);
                fput(exe_file);
        } else {
                pr_info("%s: potentially unexpected fatal signal %d.\n",
                        current->comm, signr);
        }

#if defined(__i386__) && !defined(__arch_um__)
        pr_info("code at %08lx: ", regs->ip);
        {
                int i;
                for (i = 0; i < 16; i++) {
                        unsigned char insn;

                        if (get_user(insn, (unsigned char *)(regs->ip + i)))
                                break;
                        pr_cont("%02x ", insn);
                }
        }
        pr_cont("\n");
#endif
        preempt_disable();
        show_regs(regs);
        preempt_enable();
}

static int __init setup_print_fatal_signals(char *str)
{
        get_option (&str, &print_fatal_signals);

        return 1;
}

__setup("print-fatal-signals=", setup_print_fatal_signals);

int do_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p,
                        enum pid_type type)
{
        unsigned long flags;
        int ret = -ESRCH;

        if (lock_task_sighand(p, &flags)) {
                ret = send_signal_locked(sig, info, p, type);
                unlock_task_sighand(p, &flags);
        }

        return ret;
}

enum sig_handler {
        HANDLER_CURRENT, /* If reachable use the current handler */
        HANDLER_SIG_DFL, /* Always use SIG_DFL handler semantics */
        HANDLER_EXIT,         /* Only visible as the process exit code */
};

/*
 * Force a signal that the process can't ignore: if necessary
 * we unblock the signal and change any SIG_IGN to SIG_DFL.
 *
 * Note: If we unblock the signal, we always reset it to SIG_DFL,
 * since we do not want to have a signal handler that was blocked
 * be invoked when user space had explicitly blocked it.
 *
 * We don't want to have recursive SIGSEGV's etc, for example,
 * that is why we also clear SIGNAL_UNKILLABLE.
 */
static int
force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
        enum sig_handler handler)
{
        unsigned long int flags;
        int ret, blocked, ignored;
        struct k_sigaction *action;
        int sig = info->si_signo;

        spin_lock_irqsave(&t->sighand->siglock, flags);
        action = &t->sighand->action[sig-1];
        ignored = action->sa.sa_handler == SIG_IGN;
        blocked = sigismember(&t->blocked, sig);
        if (blocked || ignored || (handler != HANDLER_CURRENT)) {
                action->sa.sa_handler = SIG_DFL;
                if (handler == HANDLER_EXIT)
                        action->sa.sa_flags |= SA_IMMUTABLE;
                if (blocked)
                        sigdelset(&t->blocked, sig);
        }
        /*
         * Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
         * debugging to leave init killable. But HANDLER_EXIT is always fatal.
         */
        if (action->sa.sa_handler == SIG_DFL &&
            (!t->ptrace || (handler == HANDLER_EXIT)))
                t->signal->flags &= ~SIGNAL_UNKILLABLE;
        ret = send_signal_locked(sig, info, t, PIDTYPE_PID);
        /* This can happen if the signal was already pending and blocked */
        if (!task_sigpending(t))
                signal_wake_up(t, 0);
        spin_unlock_irqrestore(&t->sighand->siglock, flags);

        return ret;
}

int force_sig_info(struct kernel_siginfo *info)
{
        return force_sig_info_to_task(info, current, HANDLER_CURRENT);
}

/*
 * Nuke all other threads in the group.
 */
int zap_other_threads(struct task_struct *p)
{
        struct task_struct *t;
        int count = 0;

        p->signal->group_stop_count = 0;

        for_other_threads(p, t) {
                task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
                count++;

                /* Don't bother with already dead threads */
                if (t->exit_state)
                        continue;
                sigaddset(&t->pending.signal, SIGKILL);
                signal_wake_up(t, 1);
        }

        return count;
}

struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
                                           unsigned long *flags)
{
        struct sighand_struct *sighand;

        rcu_read_lock();
        for (;;) {
                sighand = rcu_dereference(tsk->sighand);
                if (unlikely(sighand == NULL))
                        break;

                /*
                 * This sighand can be already freed and even reused, but
                 * we rely on SLAB_TYPESAFE_BY_RCU and sighand_ctor() which
                 * initializes ->siglock: this slab can't go away, it has
                 * the same object type, ->siglock can't be reinitialized.
                 *
                 * We need to ensure that tsk->sighand is still the same
                 * after we take the lock, we can race with de_thread() or
                 * __exit_signal(). In the latter case the next iteration
                 * must see ->sighand == NULL.
                 */
                spin_lock_irqsave(&sighand->siglock, *flags);
                if (likely(sighand == rcu_access_pointer(tsk->sighand)))
                        break;
                spin_unlock_irqrestore(&sighand->siglock, *flags);
        }
        rcu_read_unlock();

        return sighand;
}

#ifdef CONFIG_LOCKDEP
void lockdep_assert_task_sighand_held(struct task_struct *task)
{
        struct sighand_struct *sighand;

        rcu_read_lock();
        sighand = rcu_dereference(task->sighand);
        if (sighand)
                lockdep_assert_held(&sighand->siglock);
        else
                WARN_ON_ONCE(1);
        rcu_read_unlock();
}
#endif

/*
 * send signal info to all the members of a thread group or to the
 * individual thread if type == PIDTYPE_PID.
 */
int group_send_sig_info(int sig, struct kernel_siginfo *info,
                        struct task_struct *p, enum pid_type type)
{
        int ret;

        rcu_read_lock();
        ret = check_kill_permission(sig, info, p);
        rcu_read_unlock();

        if (!ret && sig)
                ret = do_send_sig_info(sig, info, p, type);

        return ret;
}

/*
 * __kill_pgrp_info() sends a signal to a process group: this is what the tty
 * control characters do (^C, ^Z etc)
 * - the caller must hold at least a readlock on tasklist_lock
 */
int __kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
        struct task_struct *p = NULL;
        int ret = -ESRCH;

        do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
                int err = group_send_sig_info(sig, info, p, PIDTYPE_PGID);
                /*
                 * If group_send_sig_info() succeeds at least once ret
                 * becomes 0 and after that the code below has no effect.
                 * Otherwise we return the last err or -ESRCH if this
                 * process group is empty.
                 */
                if (ret)
                        ret = err;
        } while_each_pid_task(pgrp, PIDTYPE_PGID, p);

        return ret;
}

static int kill_pid_info_type(int sig, struct kernel_siginfo *info,
                                struct pid *pid, enum pid_type type)
{
        int error = -ESRCH;
        struct task_struct *p;

        for (;;) {
                rcu_read_lock();
                p = pid_task(pid, PIDTYPE_PID);
                if (p)
                        error = group_send_sig_info(sig, info, p, type);
                rcu_read_unlock();
                if (likely(!p || error != -ESRCH))
                        return error;
                /*
                 * The task was unhashed in between, try again.  If it
                 * is dead, pid_task() will return NULL, if we race with
                 * de_thread() it will find the new leader.
                 */
        }
}

int kill_pid_info(int sig, struct kernel_siginfo *info, struct pid *pid)
{
        return kill_pid_info_type(sig, info, pid, PIDTYPE_TGID);
}

static int kill_proc_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
        int error;
        rcu_read_lock();
        error = kill_pid_info(sig, info, find_vpid(pid));
        rcu_read_unlock();
        return error;
}

static inline bool kill_as_cred_perm(const struct cred *cred,
                                     struct task_struct *target)
{
        const struct cred *pcred = __task_cred(target);

        return uid_eq(cred->euid, pcred->suid) ||
               uid_eq(cred->euid, pcred->uid) ||
               uid_eq(cred->uid, pcred->suid) ||
               uid_eq(cred->uid, pcred->uid);
}

/*
 * The usb asyncio usage of siginfo is wrong.  The glibc support
 * for asyncio which uses SI_ASYNCIO assumes the layout is SIL_RT.
 * AKA after the generic fields:
 *        kernel_pid_t        si_pid;
 *        kernel_uid32_t        si_uid;
 *        sigval_t        si_value;
 *
 * Unfortunately when usb generates SI_ASYNCIO it assumes the layout
 * after the generic fields is:
 *        void __user         *si_addr;
 *
 * This is a practical problem when there is a 64bit big endian kernel
 * and a 32bit userspace.  As the 32bit address will encoded in the low
 * 32bits of the pointer.  Those low 32bits will be stored at higher
 * address than appear in a 32 bit pointer.  So userspace will not
 * see the address it was expecting for it's completions.
 *
 * There is nothing in the encoding that can allow
 * copy_siginfo_to_user32 to detect this confusion of formats, so
 * handle this by requiring the caller of kill_pid_usb_asyncio to
 * notice when this situration takes place and to store the 32bit
 * pointer in sival_int, instead of sival_addr of the sigval_t addr
 * parameter.
 */
int kill_pid_usb_asyncio(int sig, int errno, sigval_t addr,
                         struct pid *pid, const struct cred *cred)
{
        struct kernel_siginfo info;
        struct task_struct *p;
        unsigned long flags;
        int ret = -EINVAL;

        if (!valid_signal(sig))
                return ret;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = errno;
        info.si_code = SI_ASYNCIO;
        *((sigval_t *)&info.si_pid) = addr;

        rcu_read_lock();
        p = pid_task(pid, PIDTYPE_PID);
        if (!p) {
                ret = -ESRCH;
                goto out_unlock;
        }
        if (!kill_as_cred_perm(cred, p)) {
                ret = -EPERM;
                goto out_unlock;
        }
        ret = security_task_kill(p, &info, sig, cred);
        if (ret)
                goto out_unlock;

        if (sig) {
                if (lock_task_sighand(p, &flags)) {
                        ret = __send_signal_locked(sig, &info, p, PIDTYPE_TGID, false);
                        unlock_task_sighand(p, &flags);
                } else
                        ret = -ESRCH;
        }
out_unlock:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(kill_pid_usb_asyncio);

/*
 * kill_something_info() interprets pid in interesting ways just like kill(2).
 *
 * POSIX specifies that kill(-1,sig) is unspecified, but what we have
 * is probably wrong.  Should make it like BSD or SYSV.
 */

static int kill_something_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
        int ret;

        if (pid > 0)
                return kill_proc_info(sig, info, pid);

        /* -INT_MIN is undefined.  Exclude this case to avoid a UBSAN warning */
        if (pid == INT_MIN)
                return -ESRCH;

        read_lock(&tasklist_lock);
        if (pid != -1) {
                ret = __kill_pgrp_info(sig, info,
                                pid ? find_vpid(-pid) : task_pgrp(current));
        } else {
                int retval = 0, count = 0;
                struct task_struct * p;

                for_each_process(p) {
                        if (task_pid_vnr(p) > 1 &&
                                        !same_thread_group(p, current)) {
                                int err = group_send_sig_info(sig, info, p,
                                                              PIDTYPE_MAX);
                                ++count;
                                if (err != -EPERM)
                                        retval = err;
                        }
                }
                ret = count ? retval : -ESRCH;
        }
        read_unlock(&tasklist_lock);

        return ret;
}

/*
 * These are for backward compatibility with the rest of the kernel source.
 */

int send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p)
{
        /*
         * Make sure legacy kernel users don't send in bad values
         * (normal paths check this in check_kill_permission).
         */
        if (!valid_signal(sig))
                return -EINVAL;

        return do_send_sig_info(sig, info, p, PIDTYPE_PID);
}
EXPORT_SYMBOL(send_sig_info);

#define __si_special(priv) \
        ((priv) ? SEND_SIG_PRIV : SEND_SIG_NOINFO)

int
send_sig(int sig, struct task_struct *p, int priv)
{
        return send_sig_info(sig, __si_special(priv), p);
}
EXPORT_SYMBOL(send_sig);

void force_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info(&info);
}
EXPORT_SYMBOL(force_sig);

void force_fatal_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info_to_task(&info, current, HANDLER_SIG_DFL);
}

void force_exit_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info_to_task(&info, current, HANDLER_EXIT);
}

/*
 * When things go south during signal handling, we
 * will force a SIGSEGV. And if the signal that caused
 * the problem was already a SIGSEGV, we'll want to
 * make sure we don't even try to deliver the signal..
 */
void force_sigsegv(int sig)
{
        if (sig == SIGSEGV)
                force_fatal_sig(SIGSEGV);
        else
                force_sig(SIGSEGV);
}

int force_sig_fault_to_task(int sig, int code, void __user *addr,
                            struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        return force_sig_info_to_task(&info, t, HANDLER_CURRENT);
}

int force_sig_fault(int sig, int code, void __user *addr)
{
        return force_sig_fault_to_task(sig, code, addr, current);
}

int send_sig_fault(int sig, int code, void __user *addr, struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        return send_sig_info(info.si_signo, &info, t);
}

int force_sig_mceerr(int code, void __user *addr, short lsb)
{
        struct kernel_siginfo info;

        WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
        clear_siginfo(&info);
        info.si_signo = SIGBUS;
        info.si_errno = 0;
        info.si_code = code;
        info.si_addr = addr;
        info.si_addr_lsb = lsb;
        return force_sig_info(&info);
}

int send_sig_mceerr(int code, void __user *addr, short lsb, struct task_struct *t)
{
        struct kernel_siginfo info;

        WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
        clear_siginfo(&info);
        info.si_signo = SIGBUS;
        info.si_errno = 0;
        info.si_code = code;
        info.si_addr = addr;
        info.si_addr_lsb = lsb;
        return send_sig_info(info.si_signo, &info, t);
}
EXPORT_SYMBOL(send_sig_mceerr);

int force_sig_bnderr(void __user *addr, void __user *lower, void __user *upper)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSEGV;
        info.si_errno = 0;
        info.si_code  = SEGV_BNDERR;
        info.si_addr  = addr;
        info.si_lower = lower;
        info.si_upper = upper;
        return force_sig_info(&info);
}

#ifdef SEGV_PKUERR
int force_sig_pkuerr(void __user *addr, u32 pkey)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSEGV;
        info.si_errno = 0;
        info.si_code  = SEGV_PKUERR;
        info.si_addr  = addr;
        info.si_pkey  = pkey;
        return force_sig_info(&info);
}
#endif

int send_sig_perf(void __user *addr, u32 type, u64 sig_data)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo     = SIGTRAP;
        info.si_errno     = 0;
        info.si_code      = TRAP_PERF;
        info.si_addr      = addr;
        info.si_perf_data = sig_data;
        info.si_perf_type = type;

        /*
         * Signals generated by perf events should not terminate the whole
         * process if SIGTRAP is blocked, however, delivering the signal
         * asynchronously is better than not delivering at all. But tell user
         * space if the signal was asynchronous, so it can clearly be
         * distinguished from normal synchronous ones.
         */
        info.si_perf_flags = sigismember(&current->blocked, info.si_signo) ?
                                     TRAP_PERF_FLAG_ASYNC :
                                     0;

        return send_sig_info(info.si_signo, &info, current);
}

/**
 * force_sig_seccomp - signals the task to allow in-process syscall emulation
 * @syscall: syscall number to send to userland
 * @reason: filter-supplied reason code to send to userland (via si_errno)
 * @force_coredump: true to trigger a coredump
 *
 * Forces a SIGSYS with a code of SYS_SECCOMP and related sigsys info.
 */
int force_sig_seccomp(int syscall, int reason, bool force_coredump)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSYS;
        info.si_code = SYS_SECCOMP;
        info.si_call_addr = (void __user *)KSTK_EIP(current);
        info.si_errno = reason;
        info.si_arch = syscall_get_arch(current);
        info.si_syscall = syscall;
        return force_sig_info_to_task(&info, current,
                force_coredump ? HANDLER_EXIT : HANDLER_CURRENT);
}

/* For the crazy architectures that include trap information in
 * the errno field, instead of an actual errno value.
 */
int force_sig_ptrace_errno_trap(int errno, void __user *addr)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGTRAP;
        info.si_errno = errno;
        info.si_code  = TRAP_HWBKPT;
        info.si_addr  = addr;
        return force_sig_info(&info);
}

/* For the rare architectures that include trap information using
 * si_trapno.
 */
int force_sig_fault_trapno(int sig, int code, void __user *addr, int trapno)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        info.si_trapno = trapno;
        return force_sig_info(&info);
}

/* For the rare architectures that include trap information using
 * si_trapno.
 */
int send_sig_fault_trapno(int sig, int code, void __user *addr, int trapno,
                          struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        info.si_trapno = trapno;
        return send_sig_info(info.si_signo, &info, t);
}

static int kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
        int ret;
        read_lock(&tasklist_lock);
        ret = __kill_pgrp_info(sig, info, pgrp);
        read_unlock(&tasklist_lock);
        return ret;
}

int kill_pgrp(struct pid *pid, int sig, int priv)
{
        return kill_pgrp_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pgrp);

int kill_pid(struct pid *pid, int sig, int priv)
{
        return kill_pid_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pid);

#ifdef CONFIG_POSIX_TIMERS
/*
 * These functions handle POSIX timer signals. POSIX timers use
 * preallocated sigqueue structs for sending signals.
 */
static void __flush_itimer_signals(struct sigpending *pending)
{
        sigset_t signal, retain;
        struct sigqueue *q, *n;

        signal = pending->signal;
        sigemptyset(&retain);

        list_for_each_entry_safe(q, n, &pending->list, list) {
                int sig = q->info.si_signo;

                if (likely(q->info.si_code != SI_TIMER)) {
                        sigaddset(&retain, sig);
                } else {
                        sigdelset(&signal, sig);
                        list_del_init(&q->list);
                        __sigqueue_free(q);
                }
        }

        sigorsets(&pending->signal, &signal, &retain);
}

void flush_itimer_signals(void)
{
        struct task_struct *tsk = current;

        guard(spinlock_irqsave)(&tsk->sighand->siglock);
        __flush_itimer_signals(&tsk->pending);
        __flush_itimer_signals(&tsk->signal->shared_pending);
}

bool posixtimer_init_sigqueue(struct sigqueue *q)
{
        struct ucounts *ucounts = sig_get_ucounts(current, -1, 0);

        if (!ucounts)
                return false;
        clear_siginfo(&q->info);
        __sigqueue_init(q, ucounts, SIGQUEUE_PREALLOC);
        return true;
}

static void posixtimer_queue_sigqueue(struct sigqueue *q, struct task_struct *t, enum pid_type type)
{
        struct sigpending *pending;
        int sig = q->info.si_signo;

        signalfd_notify(t, sig);
        pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
        list_add_tail(&q->list, &pending->list);
        sigaddset(&pending->signal, sig);
        complete_signal(sig, t, type);
}

/*
 * This function is used by POSIX timers to deliver a timer signal.
 * Where type is PIDTYPE_PID (such as for timers with SIGEV_THREAD_ID
 * set), the signal must be delivered to the specific thread (queues
 * into t->pending).
 *
 * Where type is not PIDTYPE_PID, signals must be delivered to the
 * process. In this case, prefer to deliver to current if it is in
 * the same thread group as the target process and its sighand is
 * stable, which avoids unnecessarily waking up a potentially idle task.
 */
static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr)
{
        struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type);

        if (t && tmr->it_pid_type != PIDTYPE_PID &&
            same_thread_group(t, current) && !current->exit_state)
                t = current;
        return t;
}

void posixtimer_send_sigqueue(struct k_itimer *tmr)
{
        struct sigqueue *q = &tmr->sigq;
        int sig = q->info.si_signo;
        struct task_struct *t;
        unsigned long flags;
        int result;

        guard(rcu)();

        t = posixtimer_get_target(tmr);
        if (!t)
                return;

        if (!likely(lock_task_sighand(t, &flags)))
                return;

        /*
         * Update @tmr::sigqueue_seq for posix timer signals with sighand
         * locked to prevent a race against dequeue_signal().
         */
        tmr->it_sigqueue_seq = tmr->it_signal_seq;

        /*
         * Set the signal delivery status under sighand lock, so that the
         * ignored signal handling can distinguish between a periodic and a
         * non-periodic timer.
         */
        tmr->it_sig_periodic = tmr->it_status == POSIX_TIMER_REQUEUE_PENDING;

        if (!prepare_signal(sig, t, false)) {
                result = TRACE_SIGNAL_IGNORED;

                if (!list_empty(&q->list)) {
                        /*
                         * The signal was ignored and blocked. The timer
                         * expiry queued it because blocked signals are
                         * queued independent of the ignored state.
                         *
                         * The unblocking set SIGPENDING, but the signal
                         * was not yet dequeued from the pending list.
                         * So prepare_signal() sees unblocked and ignored,
                         * which ends up here. Leave it queued like a
                         * regular signal.
                         *
                         * The same happens when the task group is exiting
                         * and the signal is already queued.
                         * prepare_signal() treats SIGNAL_GROUP_EXIT as
                         * ignored independent of its queued state. This
                         * gets cleaned up in __exit_signal().
                         */
                        goto out;
                }

                /* Periodic timers with SIG_IGN are queued on the ignored list */
                if (tmr->it_sig_periodic) {
                        /*
                         * Already queued means the timer was rearmed after
                         * the previous expiry got it on the ignore list.
                         * Nothing to do for that case.
                         */
                        if (hlist_unhashed(&tmr->ignored_list)) {
                                /*
                                 * Take a signal reference and queue it on
                                 * the ignored list.
                                 */
                                posixtimer_sigqueue_getref(q);
                                posixtimer_sig_ignore(t, q);
                        }
                } else if (!hlist_unhashed(&tmr->ignored_list)) {
                        /*
                         * Covers the case where a timer was periodic and
                         * then the signal was ignored. Later it was rearmed
                         * as oneshot timer. The previous signal is invalid
                         * now, and this oneshot signal has to be dropped.
                         * Remove it from the ignored list and drop the
                         * reference count as the signal is not longer
                         * queued.
                         */
                        hlist_del_init(&tmr->ignored_list);
                        posixtimer_putref(tmr);
                }
                goto out;
        }

        if (unlikely(!list_empty(&q->list))) {
                /* This holds a reference count already */
                result = TRACE_SIGNAL_ALREADY_PENDING;
                goto out;
        }

        /*
         * If the signal is on the ignore list, it got blocked after it was
         * ignored earlier. But nothing lifted the ignore. Move it back to
         * the pending list to be consistent with the regular signal
         * handling. This already holds a reference count.
         *
         * If it's not on the ignore list acquire a reference count.
         */
        if (likely(hlist_unhashed(&tmr->ignored_list)))
                posixtimer_sigqueue_getref(q);
        else
                hlist_del_init(&tmr->ignored_list);

        posixtimer_queue_sigqueue(q, t, tmr->it_pid_type);
        result = TRACE_SIGNAL_DELIVERED;
out:
        trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result);
        unlock_task_sighand(t, &flags);
}

static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q)
{
        struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);

        /*
         * If the timer is marked deleted already or the signal originates
         * from a non-periodic timer, then just drop the reference
         * count. Otherwise queue it on the ignored list.
         */
        if (posixtimer_valid(tmr) && tmr->it_sig_periodic)
                hlist_add_head(&tmr->ignored_list, &tsk->signal->ignored_posix_timers);
        else
                posixtimer_putref(tmr);
}

static void posixtimer_sig_unignore(struct task_struct *tsk, int sig)
{
        struct hlist_head *head = &tsk->signal->ignored_posix_timers;
        struct hlist_node *tmp;
        struct k_itimer *tmr;

        if (likely(hlist_empty(head)))
                return;

        /*
         * Rearming a timer with sighand lock held is not possible due to
         * lock ordering vs. tmr::it_lock. Just stick the sigqueue back and
         * let the signal delivery path deal with it whether it needs to be
         * rearmed or not. This cannot be decided here w/o dropping sighand
         * lock and creating a loop retry horror show.
         */
        hlist_for_each_entry_safe(tmr, tmp , head, ignored_list) {
                struct task_struct *target;

                /*
                 * tmr::sigq.info.si_signo is immutable, so accessing it
                 * without holding tmr::it_lock is safe.
                 */
                if (tmr->sigq.info.si_signo != sig)
                        continue;

                hlist_del_init(&tmr->ignored_list);

                /* This should never happen and leaks a reference count */
                if (WARN_ON_ONCE(!list_empty(&tmr->sigq.list)))
                        continue;

                /*
                 * Get the target for the signal. If target is a thread and
                 * has exited by now, drop the reference count.
                 */
                guard(rcu)();
                target = posixtimer_get_target(tmr);
                if (target)
                        posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type);
                else
                        posixtimer_putref(tmr);
        }
}
#else /* CONFIG_POSIX_TIMERS */
static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q) { }
static inline void posixtimer_sig_unignore(struct task_struct *tsk, int sig) { }
#endif /* !CONFIG_POSIX_TIMERS */

void do_notify_pidfd(struct task_struct *task)
{
        struct pid *pid = task_pid(task);

        WARN_ON(task->exit_state == 0);

        __wake_up(&pid->wait_pidfd, TASK_NORMAL, 0,
                        poll_to_key(EPOLLIN | EPOLLRDNORM));
}

/*
 * Let a parent know about the death of a child.
 * For a stopped/continued status change, use do_notify_parent_cldstop instead.
 *
 * Returns true if our parent ignored us and so we've switched to
 * self-reaping.
 */
bool do_notify_parent(struct task_struct *tsk, int sig)
{
        struct kernel_siginfo info;
        unsigned long flags;
        struct sighand_struct *psig;
        bool autoreap = false;
        u64 utime, stime;

        WARN_ON_ONCE(sig == -1);

        /* do_notify_parent_cldstop should have been called instead.  */
        WARN_ON_ONCE(task_is_stopped_or_traced(tsk));

        WARN_ON_ONCE(!tsk->ptrace &&
               (tsk->group_leader != tsk || !thread_group_empty(tsk)));

        /* ptraced, or group-leader without sub-threads */
        do_notify_pidfd(tsk);

        if (sig != SIGCHLD) {
                /*
                 * This is only possible if parent == real_parent.
                 * Check if it has changed security domain.
                 */
                if (tsk->parent_exec_id != READ_ONCE(tsk->parent->self_exec_id))
                        sig = SIGCHLD;
        }

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        /*
         * We are under tasklist_lock here so our parent is tied to
         * us and cannot change.
         *
         * task_active_pid_ns will always return the same pid namespace
         * until a task passes through release_task.
         *
         * write_lock() currently calls preempt_disable() which is the
         * same as rcu_read_lock(), but according to Oleg, this is not
         * correct to rely on this
         */
        rcu_read_lock();
        info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(tsk->parent));
        info.si_uid = from_kuid_munged(task_cred_xxx(tsk->parent, user_ns),
                                       task_uid(tsk));
        rcu_read_unlock();

        task_cputime(tsk, &utime, &stime);
        info.si_utime = nsec_to_clock_t(utime + tsk->signal->utime);
        info.si_stime = nsec_to_clock_t(stime + tsk->signal->stime);

        info.si_status = tsk->exit_code & 0x7f;
        if (tsk->exit_code & 0x80)
                info.si_code = CLD_DUMPED;
        else if (tsk->exit_code & 0x7f)
                info.si_code = CLD_KILLED;
        else {
                info.si_code = CLD_EXITED;
                info.si_status = tsk->exit_code >> 8;
        }

        psig = tsk->parent->sighand;
        spin_lock_irqsave(&psig->siglock, flags);
        if (!tsk->ptrace && sig == SIGCHLD &&
            (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN ||
             (psig->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDWAIT))) {
                /*
                 * We are exiting and our parent doesn't care.  POSIX.1
                 * defines special semantics for setting SIGCHLD to SIG_IGN
                 * or setting the SA_NOCLDWAIT flag: we should be reaped
                 * automatically and not left for our parent's wait4 call.
                 * Rather than having the parent do it as a magic kind of
                 * signal handler, we just set this to tell do_exit that we
                 * can be cleaned up without becoming a zombie.  Note that
                 * we still call __wake_up_parent in this case, because a
                 * blocked sys_wait4 might now return -ECHILD.
                 *
                 * Whether we send SIGCHLD or not for SA_NOCLDWAIT
                 * is implementation-defined: we do (if you don't want
                 * it, just use SIG_IGN instead).
                 */
                autoreap = true;
                if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN)
                        sig = 0;
        }
        /*
         * Send with __send_signal as si_pid and si_uid are in the
         * parent's namespaces.
         */
        if (valid_signal(sig) && sig)
                __send_signal_locked(sig, &info, tsk->parent, PIDTYPE_TGID, false);
        __wake_up_parent(tsk, tsk->parent);
        spin_unlock_irqrestore(&psig->siglock, flags);

        return autoreap;
}

/**
 * do_notify_parent_cldstop - notify parent of stopped/continued state change
 * @tsk: task reporting the state change
 * @for_ptracer: the notification is for ptracer
 * @why: CLD_{CONTINUED|STOPPED|TRAPPED} to report
 *
 * Notify @tsk's parent that the stopped/continued state has changed.  If
 * @for_ptracer is %false, @tsk's group leader notifies to its real parent.
 * If %true, @tsk reports to @tsk->parent which should be the ptracer.
 *
 * CONTEXT:
 * Must be called with tasklist_lock at least read locked.
 */
static void do_notify_parent_cldstop(struct task_struct *tsk,
                                     bool for_ptracer, int why)
{
        struct kernel_siginfo info;
        unsigned long flags;
        struct task_struct *parent;
        struct sighand_struct *sighand;
        u64 utime, stime;

        if (for_ptracer) {
                parent = tsk->parent;
        } else {
                tsk = tsk->group_leader;
                parent = tsk->real_parent;
        }

        clear_siginfo(&info);
        info.si_signo = SIGCHLD;
        info.si_errno = 0;
        /*
         * see comment in do_notify_parent() about the following 4 lines
         */
        rcu_read_lock();
        info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(parent));
        info.si_uid = from_kuid_munged(task_cred_xxx(parent, user_ns), task_uid(tsk));
        rcu_read_unlock();

        task_cputime(tsk, &utime, &stime);
        info.si_utime = nsec_to_clock_t(utime);
        info.si_stime = nsec_to_clock_t(stime);

         info.si_code = why;
         switch (why) {
         case CLD_CONTINUED:
                 info.si_status = SIGCONT;
                 break;
         case CLD_STOPPED:
                 info.si_status = tsk->signal->group_exit_code & 0x7f;
                 break;
         case CLD_TRAPPED:
                 info.si_status = tsk->exit_code & 0x7f;
                 break;
         default:
                 BUG();
         }

        sighand = parent->sighand;
        spin_lock_irqsave(&sighand->siglock, flags);
        if (sighand->action[SIGCHLD-1].sa.sa_handler != SIG_IGN &&
            !(sighand->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDSTOP))
                send_signal_locked(SIGCHLD, &info, parent, PIDTYPE_TGID);
        /*
         * Even if SIGCHLD is not generated, we must wake up wait4 calls.
         */
        __wake_up_parent(tsk, parent);
        spin_unlock_irqrestore(&sighand->siglock, flags);
}

/*
 * This must be called with current->sighand->siglock held.
 *
 * This should be the path for all ptrace stops.
 * We always set current->last_siginfo while stopped here.
 * That makes it a way to test a stopped process for
 * being ptrace-stopped vs being job-control-stopped.
 *
 * Returns the signal the ptracer requested the code resume
 * with.  If the code did not stop because the tracer is gone,
 * the stop signal remains unchanged unless clear_code.
 */
static int ptrace_stop(int exit_code, int why, unsigned long message,
                       kernel_siginfo_t *info)
        __releases(&current->sighand->siglock)
        __acquires(&current->sighand->siglock)
{
        bool gstop_done = false;

        if (arch_ptrace_stop_needed()) {
                /*
                 * The arch code has something special to do before a
                 * ptrace stop.  This is allowed to block, e.g. for faults
                 * on user stack pages.  We can't keep the siglock while
                 * calling arch_ptrace_stop, so we must release it now.
                 * To preserve proper semantics, we must do this before
                 * any signal bookkeeping like checking group_stop_count.
                 */
                spin_unlock_irq(&current->sighand->siglock);
                arch_ptrace_stop();
                spin_lock_irq(&current->sighand->siglock);
        }

        /*
         * After this point ptrace_signal_wake_up or signal_wake_up
         * will clear TASK_TRACED if ptrace_unlink happens or a fatal
         * signal comes in.  Handle previous ptrace_unlinks and fatal
         * signals here to prevent ptrace_stop sleeping in schedule.
         */
        if (!current->ptrace || __fatal_signal_pending(current))
                return exit_code;

        set_special_state(TASK_TRACED);
        current->jobctl |= JOBCTL_TRACED;

        /*
         * We're committing to trapping.  TRACED should be visible before
         * TRAPPING is cleared; otherwise, the tracer might fail do_wait().
         * Also, transition to TRACED and updates to ->jobctl should be
         * atomic with respect to siglock and should be done after the arch
         * hook as siglock is released and regrabbed across it.
         *
         *     TRACER                                    TRACEE
         *
         *     ptrace_attach()
         * [L]   wait_on_bit(JOBCTL_TRAPPING)        [S] set_special_state(TRACED)
         *     do_wait()
         *       set_current_state()                smp_wmb();
         *       ptrace_do_wait()
         *         wait_task_stopped()
         *           task_stopped_code()
         * [L]         task_is_traced()                [S] task_clear_jobctl_trapping();
         */
        smp_wmb();

        current->ptrace_message = message;
        current->last_siginfo = info;
        current->exit_code = exit_code;

        /*
         * If @why is CLD_STOPPED, we're trapping to participate in a group
         * stop.  Do the bookkeeping.  Note that if SIGCONT was delievered
         * across siglock relocks since INTERRUPT was scheduled, PENDING
         * could be clear now.  We act as if SIGCONT is received after
         * TASK_TRACED is entered - ignore it.
         */
        if (why == CLD_STOPPED && (current->jobctl & JOBCTL_STOP_PENDING))
                gstop_done = task_participate_group_stop(current);

        /* any trap clears pending STOP trap, STOP trap clears NOTIFY */
        task_clear_jobctl_pending(current, JOBCTL_TRAP_STOP);
        if (info && info->si_code >> 8 == PTRACE_EVENT_STOP)
                task_clear_jobctl_pending(current, JOBCTL_TRAP_NOTIFY);

        /* entering a trap, clear TRAPPING */
        task_clear_jobctl_trapping(current);

        spin_unlock_irq(&current->sighand->siglock);
        read_lock(&tasklist_lock);
        /*
         * Notify parents of the stop.
         *
         * While ptraced, there are two parents - the ptracer and
         * the real_parent of the group_leader.  The ptracer should
         * know about every stop while the real parent is only
         * interested in the completion of group stop.  The states
         * for the two don't interact with each other.  Notify
         * separately unless they're gonna be duplicates.
         */
        if (current->ptrace)
                do_notify_parent_cldstop(current, true, why);
        if (gstop_done && (!current->ptrace || ptrace_reparented(current)))
                do_notify_parent_cldstop(current, false, why);

        /*
         * The previous do_notify_parent_cldstop() invocation woke ptracer.
         * One a PREEMPTION kernel this can result in preemption requirement
         * which will be fulfilled after read_unlock() and the ptracer will be
         * put on the CPU.
         * The ptracer is in wait_task_inactive(, __TASK_TRACED) waiting for
         * this task wait in schedule(). If this task gets preempted then it
         * remains enqueued on the runqueue. The ptracer will observe this and
         * then sleep for a delay of one HZ tick. In the meantime this task
         * gets scheduled, enters schedule() and will wait for the ptracer.
         *
         * This preemption point is not bad from a correctness point of
         * view but extends the runtime by one HZ tick time due to the
         * ptracer's sleep.  The preempt-disable section ensures that there
         * will be no preemption between unlock and schedule() and so
         * improving the performance since the ptracer will observe that
         * the tracee is scheduled out once it gets on the CPU.
         *
         * On PREEMPT_RT locking tasklist_lock does not disable preemption.
         * Therefore the task can be preempted after do_notify_parent_cldstop()
         * before unlocking tasklist_lock so there is no benefit in doing this.
         *
         * In fact disabling preemption is harmful on PREEMPT_RT because
         * the spinlock_t in cgroup_enter_frozen() must not be acquired
         * with preemption disabled due to the 'sleeping' spinlock
         * substitution of RT.
         */
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_disable();
        read_unlock(&tasklist_lock);
        cgroup_enter_frozen();
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable_no_resched();
        schedule();
        cgroup_leave_frozen(true);

        /*
         * We are back.  Now reacquire the siglock before touching
         * last_siginfo, so that we are sure to have synchronized with
         * any signal-sending on another CPU that wants to examine it.
         */
        spin_lock_irq(&current->sighand->siglock);
        exit_code = current->exit_code;
        current->last_siginfo = NULL;
        current->ptrace_message = 0;
        current->exit_code = 0;

        /* LISTENING can be set only during STOP traps, clear it */
        current->jobctl &= ~(JOBCTL_LISTENING | JOBCTL_PTRACE_FROZEN);

        /*
         * Queued signals ignored us while we were stopped for tracing.
         * So check for any that we should take before resuming user mode.
         * This sets TIF_SIGPENDING, but never clears it.
         */
        recalc_sigpending_tsk(current);
        return exit_code;
}

static int ptrace_do_notify(int signr, int exit_code, int why, unsigned long message)
{
        kernel_siginfo_t info;

        clear_siginfo(&info);
        info.si_signo = signr;
        info.si_code = exit_code;
        info.si_pid = task_pid_vnr(current);
        info.si_uid = from_kuid_munged(current_user_ns(), current_uid());

        /* Let the debugger run.  */
        return ptrace_stop(exit_code, why, message, &info);
}

int ptrace_notify(int exit_code, unsigned long message)
{
        int signr;

        BUG_ON((exit_code & (0x7f | ~0xffff)) != SIGTRAP);
        if (unlikely(task_work_pending(current)))
                task_work_run();

        spin_lock_irq(&current->sighand->siglock);
        signr = ptrace_do_notify(SIGTRAP, exit_code, CLD_TRAPPED, message);
        spin_unlock_irq(&current->sighand->siglock);
        return signr;
}

/**
 * do_signal_stop - handle group stop for SIGSTOP and other stop signals
 * @signr: signr causing group stop if initiating
 *
 * If %JOBCTL_STOP_PENDING is not set yet, initiate group stop with @signr
 * and participate in it.  If already set, participate in the existing
 * group stop.  If participated in a group stop (and thus slept), %true is
 * returned with siglock released.
 *
 * If ptraced, this function doesn't handle stop itself.  Instead,
 * %JOBCTL_TRAP_STOP is scheduled and %false is returned with siglock
 * untouched.  The caller must ensure that INTERRUPT trap handling takes
 * places afterwards.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held, which is released
 * on %true return.
 *
 * RETURNS:
 * %false if group stop is already cancelled or ptrace trap is scheduled.
 * %true if participated in group stop.
 */
static bool do_signal_stop(int signr)
        __releases(&current->sighand->siglock)
{
        struct signal_struct *sig = current->signal;

        if (!(current->jobctl & JOBCTL_STOP_PENDING)) {
                unsigned long gstop = JOBCTL_STOP_PENDING | JOBCTL_STOP_CONSUME;
                struct task_struct *t;

                /* signr will be recorded in task->jobctl for retries */
                WARN_ON_ONCE(signr & ~JOBCTL_STOP_SIGMASK);

                if (!likely(current->jobctl & JOBCTL_STOP_DEQUEUED) ||
                    unlikely(sig->flags & SIGNAL_GROUP_EXIT) ||
                    unlikely(sig->group_exec_task))
                        return false;
                /*
                 * There is no group stop already in progress.  We must
                 * initiate one now.
                 *
                 * While ptraced, a task may be resumed while group stop is
                 * still in effect and then receive a stop signal and
                 * initiate another group stop.  This deviates from the
                 * usual behavior as two consecutive stop signals can't
                 * cause two group stops when !ptraced.  That is why we
                 * also check !task_is_stopped(t) below.
                 *
                 * The condition can be distinguished by testing whether
                 * SIGNAL_STOP_STOPPED is already set.  Don't generate
                 * group_exit_code in such case.
                 *
                 * This is not necessary for SIGNAL_STOP_CONTINUED because
                 * an intervening stop signal is required to cause two
                 * continued events regardless of ptrace.
                 */
                if (!(sig->flags & SIGNAL_STOP_STOPPED))
                        sig->group_exit_code = signr;

                sig->group_stop_count = 0;
                if (task_set_jobctl_pending(current, signr | gstop))
                        sig->group_stop_count++;

                for_other_threads(current, t) {
                        /*
                         * Setting state to TASK_STOPPED for a group
                         * stop is always done with the siglock held,
                         * so this check has no races.
                         */
                        if (!task_is_stopped(t) &&
                            task_set_jobctl_pending(t, signr | gstop)) {
                                sig->group_stop_count++;
                                if (likely(!(t->ptrace & PT_SEIZED)))
                                        signal_wake_up(t, 0);
                                else
                                        ptrace_trap_notify(t);
                        }
                }
        }

        if (likely(!current->ptrace)) {
                int notify = 0;

                /*
                 * If there are no other threads in the group, or if there
                 * is a group stop in progress and we are the last to stop,
                 * report to the parent.
                 */
                if (task_participate_group_stop(current))
                        notify = CLD_STOPPED;

                current->jobctl |= JOBCTL_STOPPED;
                set_special_state(TASK_STOPPED);
                spin_unlock_irq(&current->sighand->siglock);

                /*
                 * Notify the parent of the group stop completion.  Because
                 * we're not holding either the siglock or tasklist_lock
                 * here, ptracer may attach inbetween; however, this is for
                 * group stop and should always be delivered to the real
                 * parent of the group leader.  The new ptracer will get
                 * its notification when this task transitions into
                 * TASK_TRACED.
                 */
                if (notify) {
                        read_lock(&tasklist_lock);
                        do_notify_parent_cldstop(current, false, notify);
                        read_unlock(&tasklist_lock);
                }

                /* Now we don't run again until woken by SIGCONT or SIGKILL */
                cgroup_enter_frozen();
                schedule();
                return true;
        } else {
                /*
                 * While ptraced, group stop is handled by STOP trap.
                 * Schedule it and let the caller deal with it.
                 */
                task_set_jobctl_pending(current, JOBCTL_TRAP_STOP);
                return false;
        }
}

/**
 * do_jobctl_trap - take care of ptrace jobctl traps
 *
 * When PT_SEIZED, it's used for both group stop and explicit
 * SEIZE/INTERRUPT traps.  Both generate PTRACE_EVENT_STOP trap with
 * accompanying siginfo.  If stopped, lower eight bits of exit_code contain
 * the stop signal; otherwise, %SIGTRAP.
 *
 * When !PT_SEIZED, it's used only for group stop trap with stop signal
 * number as exit_code and no siginfo.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held, which may be
 * released and re-acquired before returning with intervening sleep.
 */
static void do_jobctl_trap(void)
{
        struct signal_struct *signal = current->signal;
        int signr = current->jobctl & JOBCTL_STOP_SIGMASK;

        if (current->ptrace & PT_SEIZED) {
                if (!signal->group_stop_count &&
                    !(signal->flags & SIGNAL_STOP_STOPPED))
                        signr = SIGTRAP;
                WARN_ON_ONCE(!signr);
                ptrace_do_notify(signr, signr | (PTRACE_EVENT_STOP << 8),
                                 CLD_STOPPED, 0);
        } else {
                WARN_ON_ONCE(!signr);
                ptrace_stop(signr, CLD_STOPPED, 0, NULL);
        }
}

/**
 * do_freezer_trap - handle the freezer jobctl trap
 *
 * Puts the task into frozen state, if only the task is not about to quit.
 * In this case it drops JOBCTL_TRAP_FREEZE.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held,
 * which is always released before returning.
 */
static void do_freezer_trap(void)
        __releases(&current->sighand->siglock)
{
        /*
         * If there are other trap bits pending except JOBCTL_TRAP_FREEZE,
         * let's make another loop to give it a chance to be handled.
         * In any case, we'll return back.
         */
        if ((current->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) !=
             JOBCTL_TRAP_FREEZE) {
                spin_unlock_irq(&current->sighand->siglock);
                return;
        }

        /*
         * Now we're sure that there is no pending fatal signal and no
         * pending traps. Clear TIF_SIGPENDING to not get out of schedule()
         * immediately (if there is a non-fatal signal pending), and
         * put the task into sleep.
         */
        __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
        clear_thread_flag(TIF_SIGPENDING);
        spin_unlock_irq(&current->sighand->siglock);
        cgroup_enter_frozen();
        schedule();

        /*
         * We could've been woken by task_work, run it to clear
         * TIF_NOTIFY_SIGNAL. The caller will retry if necessary.
         */
        clear_notify_signal();
        if (unlikely(task_work_pending(current)))
                task_work_run();
}

static int ptrace_signal(int signr, kernel_siginfo_t *info, enum pid_type type)
{
        /*
         * We do not check sig_kernel_stop(signr) but set this marker
         * unconditionally because we do not know whether debugger will
         * change signr. This flag has no meaning unless we are going
         * to stop after return from ptrace_stop(). In this case it will
         * be checked in do_signal_stop(), we should only stop if it was
         * not cleared by SIGCONT while we were sleeping. See also the
         * comment in dequeue_signal().
         */
        current->jobctl |= JOBCTL_STOP_DEQUEUED;
        signr = ptrace_stop(signr, CLD_TRAPPED, 0, info);

        /* We're back.  Did the debugger cancel the sig?  */
        if (signr == 0)
                return signr;

        /*
         * Update the siginfo structure if the signal has
         * changed.  If the debugger wanted something
         * specific in the siginfo structure then it should
         * have updated *info via PTRACE_SETSIGINFO.
         */
        if (signr != info->si_signo) {
                clear_siginfo(info);
                info->si_signo = signr;
                info->si_errno = 0;
                info->si_code = SI_USER;
                rcu_read_lock();
                info->si_pid = task_pid_vnr(current->parent);
                info->si_uid = from_kuid_munged(current_user_ns(),
                                                task_uid(current->parent));
                rcu_read_unlock();
        }

        /* If the (new) signal is now blocked, requeue it.  */
        if (sigismember(&current->blocked, signr) ||
            fatal_signal_pending(current)) {
                send_signal_locked(signr, info, current, type);
                signr = 0;
        }

        return signr;
}

static void hide_si_addr_tag_bits(struct ksignal *ksig)
{
        switch (siginfo_layout(ksig->sig, ksig->info.si_code)) {
        case SIL_FAULT:
        case SIL_FAULT_TRAPNO:
        case SIL_FAULT_MCEERR:
        case SIL_FAULT_BNDERR:
        case SIL_FAULT_PKUERR:
        case SIL_FAULT_PERF_EVENT:
                ksig->info.si_addr = arch_untagged_si_addr(
                        ksig->info.si_addr, ksig->sig, ksig->info.si_code);
                break;
        case SIL_KILL:
        case SIL_TIMER:
        case SIL_POLL:
        case SIL_CHLD:
        case SIL_RT:
        case SIL_SYS:
                break;
        }
}

bool get_signal(struct ksignal *ksig)
{
        struct sighand_struct *sighand = current->sighand;
        struct signal_struct *signal = current->signal;
        int signr;

        clear_notify_signal();
        if (unlikely(task_work_pending(current)))
                task_work_run();

        if (!task_sigpending(current))
                return false;

        if (unlikely(uprobe_deny_signal()))
                return false;

        /*
         * Do this once, we can't return to user-mode if freezing() == T.
         * do_signal_stop() and ptrace_stop() do freezable_schedule() and
         * thus do not need another check after return.
         */
        try_to_freeze();

relock:
        spin_lock_irq(&sighand->siglock);

        /*
         * Every stopped thread goes here after wakeup. Check to see if
         * we should notify the parent, prepare_signal(SIGCONT) encodes
         * the CLD_ si_code into SIGNAL_CLD_MASK bits.
         */
        if (unlikely(signal->flags & SIGNAL_CLD_MASK)) {
                int why;

                if (signal->flags & SIGNAL_CLD_CONTINUED)
                        why = CLD_CONTINUED;
                else
                        why = CLD_STOPPED;

                signal->flags &= ~SIGNAL_CLD_MASK;

                spin_unlock_irq(&sighand->siglock);

                /*
                 * Notify the parent that we're continuing.  This event is
                 * always per-process and doesn't make whole lot of sense
                 * for ptracers, who shouldn't consume the state via
                 * wait(2) either, but, for backward compatibility, notify
                 * the ptracer of the group leader too unless it's gonna be
                 * a duplicate.
                 */
                read_lock(&tasklist_lock);
                do_notify_parent_cldstop(current, false, why);

                if (ptrace_reparented(current->group_leader))
                        do_notify_parent_cldstop(current->group_leader,
                                                true, why);
                read_unlock(&tasklist_lock);

                goto relock;
        }

        for (;;) {
                struct k_sigaction *ka;
                enum pid_type type;

                /* Has this task already been marked for death? */
                if ((signal->flags & SIGNAL_GROUP_EXIT) ||
                     signal->group_exec_task) {
                        signr = SIGKILL;
                        sigdelset(&current->pending.signal, SIGKILL);
                        trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
                                             &sighand->action[SIGKILL-1]);
                        recalc_sigpending();
                        /*
                         * implies do_group_exit() or return to PF_USER_WORKER,
                         * no need to initialize ksig->info/etc.
                         */
                        goto fatal;
                }

                if (unlikely(current->jobctl & JOBCTL_STOP_PENDING) &&
                    do_signal_stop(0))
                        goto relock;

                if (unlikely(current->jobctl &
                             (JOBCTL_TRAP_MASK | JOBCTL_TRAP_FREEZE))) {
                        if (current->jobctl & JOBCTL_TRAP_MASK) {
                                do_jobctl_trap();
                                spin_unlock_irq(&sighand->siglock);
                        } else if (current->jobctl & JOBCTL_TRAP_FREEZE)
                                do_freezer_trap();

                        goto relock;
                }

                /*
                 * If the task is leaving the frozen state, let's update
                 * cgroup counters and reset the frozen bit.
                 */
                if (unlikely(cgroup_task_frozen(current))) {
                        spin_unlock_irq(&sighand->siglock);
                        cgroup_leave_frozen(false);
                        goto relock;
                }

                /*
                 * Signals generated by the execution of an instruction
                 * need to be delivered before any other pending signals
                 * so that the instruction pointer in the signal stack
                 * frame points to the faulting instruction.
                 */
                type = PIDTYPE_PID;
                signr = dequeue_synchronous_signal(&ksig->info);
                if (!signr)
                        signr = dequeue_signal(&current->blocked, &ksig->info, &type);

                if (!signr)
                        break; /* will return 0 */

                if (unlikely(current->ptrace) && (signr != SIGKILL) &&
                    !(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) {
                        signr = ptrace_signal(signr, &ksig->info, type);
                        if (!signr)
                                continue;
                }

                ka = &sighand->action[signr-1];

                /* Trace actually delivered signals. */
                trace_signal_deliver(signr, &ksig->info, ka);

                if (ka->sa.sa_handler == SIG_IGN) /* Do nothing.  */
                        continue;
                if (ka->sa.sa_handler != SIG_DFL) {
                        /* Run the handler.  */
                        ksig->ka = *ka;

                        if (ka->sa.sa_flags & SA_ONESHOT)
                                ka->sa.sa_handler = SIG_DFL;

                        break; /* will return non-zero "signr" value */
                }

                /*
                 * Now we are doing the default action for this signal.
                 */
                if (sig_kernel_ignore(signr)) /* Default is nothing. */
                        continue;

                /*
                 * Global init gets no signals it doesn't want.
                 * Container-init gets no signals it doesn't want from same
                 * container.
                 *
                 * Note that if global/container-init sees a sig_kernel_only()
                 * signal here, the signal must have been generated internally
                 * or must have come from an ancestor namespace. In either
                 * case, the signal cannot be dropped.
                 */
                if (unlikely(signal->flags & SIGNAL_UNKILLABLE) &&
                                !sig_kernel_only(signr))
                        continue;

                if (sig_kernel_stop(signr)) {
                        /*
                         * The default action is to stop all threads in
                         * the thread group.  The job control signals
                         * do nothing in an orphaned pgrp, but SIGSTOP
                         * always works.  Note that siglock needs to be
                         * dropped during the call to is_orphaned_pgrp()
                         * because of lock ordering with tasklist_lock.
                         * This allows an intervening SIGCONT to be posted.
                         * We need to check for that and bail out if necessary.
                         */
                        if (signr != SIGSTOP) {
                                spin_unlock_irq(&sighand->siglock);

                                /* signals can be posted during this window */

                                if (is_current_pgrp_orphaned())
                                        goto relock;

                                spin_lock_irq(&sighand->siglock);
                        }

                        if (likely(do_signal_stop(signr))) {
                                /* It released the siglock.  */
                                goto relock;
                        }

                        /*
                         * We didn't actually stop, due to a race
                         * with SIGCONT or something like that.
                         */
                        continue;
                }

        fatal:
                spin_unlock_irq(&sighand->siglock);
                if (unlikely(cgroup_task_frozen(current)))
                        cgroup_leave_frozen(true);

                /*
                 * Anything else is fatal, maybe with a core dump.
                 */
                current->flags |= PF_SIGNALED;

                if (sig_kernel_coredump(signr)) {
                        if (print_fatal_signals)
                                print_fatal_signal(signr);
                        proc_coredump_connector(current);
                        /*
                         * If it was able to dump core, this kills all
                         * other threads in the group and synchronizes with
                         * their demise.  If we lost the race with another
                         * thread getting here, it set group_exit_code
                         * first and our do_group_exit call below will use
                         * that value and ignore the one we pass it.
                         */
                        do_coredump(&ksig->info);
                }

                /*
                 * PF_USER_WORKER threads will catch and exit on fatal signals
                 * themselves. They have cleanup that must be performed, so we
                 * cannot call do_exit() on their behalf. Note that ksig won't
                 * be properly initialized, PF_USER_WORKER's shouldn't use it.
                 */
                if (current->flags & PF_USER_WORKER)
                        goto out;

                /*
                 * Death signals, no core dump.
                 */
                do_group_exit(signr);
                /* NOTREACHED */
        }
        spin_unlock_irq(&sighand->siglock);

        ksig->sig = signr;

        if (signr && !(ksig->ka.sa.sa_flags & SA_EXPOSE_TAGBITS))
                hide_si_addr_tag_bits(ksig);
out:
        return signr > 0;
}

/**
 * signal_delivered - called after signal delivery to update blocked signals
 * @ksig:                kernel signal struct
 * @stepping:                nonzero if debugger single-step or block-step in use
 *
 * This function should be called when a signal has successfully been
 * delivered. It updates the blocked signals accordingly (@ksig->ka.sa.sa_mask
 * is always blocked), and the signal itself is blocked unless %SA_NODEFER
 * is set in @ksig->ka.sa.sa_flags.  Tracing is notified.
 */
static void signal_delivered(struct ksignal *ksig, int stepping)
{
        sigset_t blocked;

        /* A signal was successfully delivered, and the
           saved sigmask was stored on the signal frame,
           and will be restored by sigreturn.  So we can
           simply clear the restore sigmask flag.  */
        clear_restore_sigmask();

        sigorsets(&blocked, &current->blocked, &ksig->ka.sa.sa_mask);
        if (!(ksig->ka.sa.sa_flags & SA_NODEFER))
                sigaddset(&blocked, ksig->sig);
        set_current_blocked(&blocked);
        if (current->sas_ss_flags & SS_AUTODISARM)
                sas_ss_reset(current);
        if (stepping)
                ptrace_notify(SIGTRAP, 0);
}

void signal_setup_done(int failed, struct ksignal *ksig, int stepping)
{
        if (failed)
                force_sigsegv(ksig->sig);
        else
                signal_delivered(ksig, stepping);
}

/*
 * It could be that complete_signal() picked us to notify about the
 * group-wide signal. Other threads should be notified now to take
 * the shared signals in @which since we will not.
 */
static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
{
        sigset_t retarget;
        struct task_struct *t;

        sigandsets(&retarget, &tsk->signal->shared_pending.signal, which);
        if (sigisemptyset(&retarget))
                return;

        for_other_threads(tsk, t) {
                if (t->flags & PF_EXITING)
                        continue;

                if (!has_pending_signals(&retarget, &t->blocked))
                        continue;
                /* Remove the signals this thread can handle. */
                sigandsets(&retarget, &retarget, &t->blocked);

                if (!task_sigpending(t))
                        signal_wake_up(t, 0);

                if (sigisemptyset(&retarget))
                        break;
        }
}

void exit_signals(struct task_struct *tsk)
{
        int group_stop = 0;
        sigset_t unblocked;

        /*
         * @tsk is about to have PF_EXITING set - lock out users which
         * expect stable threadgroup.
         */
        cgroup_threadgroup_change_begin(tsk);

        if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
                sched_mm_cid_exit_signals(tsk);
                tsk->flags |= PF_EXITING;
                cgroup_threadgroup_change_end(tsk);
                return;
        }

        spin_lock_irq(&tsk->sighand->siglock);
        /*
         * From now this task is not visible for group-wide signals,
         * see wants_signal(), do_signal_stop().
         */
        sched_mm_cid_exit_signals(tsk);
        tsk->flags |= PF_EXITING;

        cgroup_threadgroup_change_end(tsk);

        if (!task_sigpending(tsk))
                goto out;

        unblocked = tsk->blocked;
        signotset(&unblocked);
        retarget_shared_pending(tsk, &unblocked);

        if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) &&
            task_participate_group_stop(tsk))
                group_stop = CLD_STOPPED;
out:
        spin_unlock_irq(&tsk->sighand->siglock);

        /*
         * If group stop has completed, deliver the notification.  This
         * should always go to the real parent of the group leader.
         */
        if (unlikely(group_stop)) {
                read_lock(&tasklist_lock);
                do_notify_parent_cldstop(tsk, false, group_stop);
                read_unlock(&tasklist_lock);
        }
}

/*
 * System call entry points.
 */

/**
 *  sys_restart_syscall - restart a system call
 */
SYSCALL_DEFINE0(restart_syscall)
{
        struct restart_block *restart = &current->restart_block;
        return restart->fn(restart);
}

long do_no_restart_syscall(struct restart_block *param)
{
        return -EINTR;
}

static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset)
{
        if (task_sigpending(tsk) && !thread_group_empty(tsk)) {
                sigset_t newblocked;
                /* A set of now blocked but previously unblocked signals. */
                sigandnsets(&newblocked, newset, &current->blocked);
                retarget_shared_pending(tsk, &newblocked);
        }
        tsk->blocked = *newset;
        recalc_sigpending();
}

/**
 * set_current_blocked - change current->blocked mask
 * @newset: new mask
 *
 * It is wrong to change ->blocked directly, this helper should be used
 * to ensure the process can't miss a shared signal we are going to block.
 */
void set_current_blocked(sigset_t *newset)
{
        sigdelsetmask(newset, sigmask(SIGKILL) | sigmask(SIGSTOP));
        __set_current_blocked(newset);
}

void __set_current_blocked(const sigset_t *newset)
{
        struct task_struct *tsk = current;

        /*
         * In case the signal mask hasn't changed, there is nothing we need
         * to do. The current->blocked shouldn't be modified by other task.
         */
        if (sigequalsets(&tsk->blocked, newset))
                return;

        spin_lock_irq(&tsk->sighand->siglock);
        __set_task_blocked(tsk, newset);
        spin_unlock_irq(&tsk->sighand->siglock);
}

/*
 * This is also useful for kernel threads that want to temporarily
 * (or permanently) block certain signals.
 *
 * NOTE! Unlike the user-mode sys_sigprocmask(), the kernel
 * interface happily blocks "unblockable" signals like SIGKILL
 * and friends.
 */
int sigprocmask(int how, sigset_t *set, sigset_t *oldset)
{
        struct task_struct *tsk = current;
        sigset_t newset;

        /* Lockless, only current can change ->blocked, never from irq */
        if (oldset)
                *oldset = tsk->blocked;

        switch (how) {
        case SIG_BLOCK:
                sigorsets(&newset, &tsk->blocked, set);
                break;
        case SIG_UNBLOCK:
                sigandnsets(&newset, &tsk->blocked, set);
                break;
        case SIG_SETMASK:
                newset = *set;
                break;
        default:
                return -EINVAL;
        }

        __set_current_blocked(&newset);
        return 0;
}
EXPORT_SYMBOL(sigprocmask);

/*
 * The api helps set app-provided sigmasks.
 *
 * This is useful for syscalls such as ppoll, pselect, io_pgetevents and
 * epoll_pwait where a new sigmask is passed from userland for the syscalls.
 *
 * Note that it does set_restore_sigmask() in advance, so it must be always
 * paired with restore_saved_sigmask_unless() before return from syscall.
 */
int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize)
{
        sigset_t kmask;

        if (!umask)
                return 0;
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;
        if (copy_from_user(&kmask, umask, sizeof(sigset_t)))
                return -EFAULT;

        set_restore_sigmask();
        current->saved_sigmask = current->blocked;
        set_current_blocked(&kmask);

        return 0;
}

#ifdef CONFIG_COMPAT
int set_compat_user_sigmask(const compat_sigset_t __user *umask,
                            size_t sigsetsize)
{
        sigset_t kmask;

        if (!umask)
                return 0;
        if (sigsetsize != sizeof(compat_sigset_t))
                return -EINVAL;
        if (get_compat_sigset(&kmask, umask))
                return -EFAULT;

        set_restore_sigmask();
        current->saved_sigmask = current->blocked;
        set_current_blocked(&kmask);

        return 0;
}
#endif

/**
 *  sys_rt_sigprocmask - change the list of currently blocked signals
 *  @how: whether to add, remove, or set signals
 *  @nset: stores pending signals
 *  @oset: previous value of signal mask if non-null
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigprocmask, int, how, sigset_t __user *, nset,
                sigset_t __user *, oset, size_t, sigsetsize)
{
        sigset_t old_set, new_set;
        int error;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        old_set = current->blocked;

        if (nset) {
                if (copy_from_user(&new_set, nset, sizeof(sigset_t)))
                        return -EFAULT;
                sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));

                error = sigprocmask(how, &new_set, NULL);
                if (error)
                        return error;
        }

        if (oset) {
                if (copy_to_user(oset, &old_set, sizeof(sigset_t)))
                        return -EFAULT;
        }

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigprocmask, int, how, compat_sigset_t __user *, nset,
                compat_sigset_t __user *, oset, compat_size_t, sigsetsize)
{
        sigset_t old_set = current->blocked;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (nset) {
                sigset_t new_set;
                int error;
                if (get_compat_sigset(&new_set, nset))
                        return -EFAULT;
                sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));

                error = sigprocmask(how, &new_set, NULL);
                if (error)
                        return error;
        }
        return oset ? put_compat_sigset(oset, &old_set, sizeof(*oset)) : 0;
}
#endif

static void do_sigpending(sigset_t *set)
{
        spin_lock_irq(&current->sighand->siglock);
        sigorsets(set, &current->pending.signal,
                  &current->signal->shared_pending.signal);
        spin_unlock_irq(&current->sighand->siglock);

        /* Outside the lock because only this thread touches it.  */
        sigandsets(set, &current->blocked, set);
}

/**
 *  sys_rt_sigpending - examine a pending signal that has been raised
 *                        while blocked
 *  @uset: stores pending signals
 *  @sigsetsize: size of sigset_t type or larger
 */
SYSCALL_DEFINE2(rt_sigpending, sigset_t __user *, uset, size_t, sigsetsize)
{
        sigset_t set;

        if (sigsetsize > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        if (copy_to_user(uset, &set, sigsetsize))
                return -EFAULT;

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigpending, compat_sigset_t __user *, uset,
                compat_size_t, sigsetsize)
{
        sigset_t set;

        if (sigsetsize > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        return put_compat_sigset(uset, &set, sigsetsize);
}
#endif

static const struct {
        unsigned char limit, layout;
} sig_sicodes[] = {
        [SIGILL]  = { NSIGILL,  SIL_FAULT },
        [SIGFPE]  = { NSIGFPE,  SIL_FAULT },
        [SIGSEGV] = { NSIGSEGV, SIL_FAULT },
        [SIGBUS]  = { NSIGBUS,  SIL_FAULT },
        [SIGTRAP] = { NSIGTRAP, SIL_FAULT },
#if defined(SIGEMT)
        [SIGEMT]  = { NSIGEMT,  SIL_FAULT },
#endif
        [SIGCHLD] = { NSIGCHLD, SIL_CHLD },
        [SIGPOLL] = { NSIGPOLL, SIL_POLL },
        [SIGSYS]  = { NSIGSYS,  SIL_SYS },
};

static bool known_siginfo_layout(unsigned sig, int si_code)
{
        if (si_code == SI_KERNEL)
                return true;
        else if ((si_code > SI_USER)) {
                if (sig_specific_sicodes(sig)) {
                        if (si_code <= sig_sicodes[sig].limit)
                                return true;
                }
                else if (si_code <= NSIGPOLL)
                        return true;
        }
        else if (si_code >= SI_DETHREAD)
                return true;
        else if (si_code == SI_ASYNCNL)
                return true;
        return false;
}

enum siginfo_layout siginfo_layout(unsigned sig, int si_code)
{
        enum siginfo_layout layout = SIL_KILL;
        if ((si_code > SI_USER) && (si_code < SI_KERNEL)) {
                if ((sig < ARRAY_SIZE(sig_sicodes)) &&
                    (si_code <= sig_sicodes[sig].limit)) {
                        layout = sig_sicodes[sig].layout;
                        /* Handle the exceptions */
                        if ((sig == SIGBUS) &&
                            (si_code >= BUS_MCEERR_AR) && (si_code <= BUS_MCEERR_AO))
                                layout = SIL_FAULT_MCEERR;
                        else if ((sig == SIGSEGV) && (si_code == SEGV_BNDERR))
                                layout = SIL_FAULT_BNDERR;
#ifdef SEGV_PKUERR
                        else if ((sig == SIGSEGV) && (si_code == SEGV_PKUERR))
                                layout = SIL_FAULT_PKUERR;
#endif
                        else if ((sig == SIGTRAP) && (si_code == TRAP_PERF))
                                layout = SIL_FAULT_PERF_EVENT;
                        else if (IS_ENABLED(CONFIG_SPARC) &&
                                 (sig == SIGILL) && (si_code == ILL_ILLTRP))
                                layout = SIL_FAULT_TRAPNO;
                        else if (IS_ENABLED(CONFIG_ALPHA) &&
                                 ((sig == SIGFPE) ||
                                  ((sig == SIGTRAP) && (si_code == TRAP_UNK))))
                                layout = SIL_FAULT_TRAPNO;
                }
                else if (si_code <= NSIGPOLL)
                        layout = SIL_POLL;
        } else {
                if (si_code == SI_TIMER)
                        layout = SIL_TIMER;
                else if (si_code == SI_SIGIO)
                        layout = SIL_POLL;
                else if (si_code < 0)
                        layout = SIL_RT;
        }
        return layout;
}

static inline char __user *si_expansion(const siginfo_t __user *info)
{
        return ((char __user *)info) + sizeof(struct kernel_siginfo);
}

int copy_siginfo_to_user(siginfo_t __user *to, const kernel_siginfo_t *from)
{
        char __user *expansion = si_expansion(to);
        if (copy_to_user(to, from , sizeof(struct kernel_siginfo)))
                return -EFAULT;
        if (clear_user(expansion, SI_EXPANSION_SIZE))
                return -EFAULT;
        return 0;
}

static int post_copy_siginfo_from_user(kernel_siginfo_t *info,
                                       const siginfo_t __user *from)
{
        if (unlikely(!known_siginfo_layout(info->si_signo, info->si_code))) {
                char __user *expansion = si_expansion(from);
                char buf[SI_EXPANSION_SIZE];
                int i;
                /*
                 * An unknown si_code might need more than
                 * sizeof(struct kernel_siginfo) bytes.  Verify all of the
                 * extra bytes are 0.  This guarantees copy_siginfo_to_user
                 * will return this data to userspace exactly.
                 */
                if (copy_from_user(&buf, expansion, SI_EXPANSION_SIZE))
                        return -EFAULT;
                for (i = 0; i < SI_EXPANSION_SIZE; i++) {
                        if (buf[i] != 0)
                                return -E2BIG;
                }
        }
        return 0;
}

static int __copy_siginfo_from_user(int signo, kernel_siginfo_t *to,
                                    const siginfo_t __user *from)
{
        if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
                return -EFAULT;
        to->si_signo = signo;
        return post_copy_siginfo_from_user(to, from);
}

int copy_siginfo_from_user(kernel_siginfo_t *to, const siginfo_t __user *from)
{
        if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
                return -EFAULT;
        return post_copy_siginfo_from_user(to, from);
}

#ifdef CONFIG_COMPAT
/**
 * copy_siginfo_to_external32 - copy a kernel siginfo into a compat user siginfo
 * @to: compat siginfo destination
 * @from: kernel siginfo source
 *
 * Note: This function does not work properly for the SIGCHLD on x32, but
 * fortunately it doesn't have to.  The only valid callers for this function are
 * copy_siginfo_to_user32, which is overriden for x32 and the coredump code.
 * The latter does not care because SIGCHLD will never cause a coredump.
 */
void copy_siginfo_to_external32(struct compat_siginfo *to,
                const struct kernel_siginfo *from)
{
        memset(to, 0, sizeof(*to));

        to->si_signo = from->si_signo;
        to->si_errno = from->si_errno;
        to->si_code  = from->si_code;
        switch(siginfo_layout(from->si_signo, from->si_code)) {
        case SIL_KILL:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                break;
        case SIL_TIMER:
                to->si_tid     = from->si_tid;
                to->si_overrun = from->si_overrun;
                to->si_int     = from->si_int;
                break;
        case SIL_POLL:
                to->si_band = from->si_band;
                to->si_fd   = from->si_fd;
                break;
        case SIL_FAULT:
                to->si_addr = ptr_to_compat(from->si_addr);
                break;
        case SIL_FAULT_TRAPNO:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_trapno = from->si_trapno;
                break;
        case SIL_FAULT_MCEERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_addr_lsb = from->si_addr_lsb;
                break;
        case SIL_FAULT_BNDERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_lower = ptr_to_compat(from->si_lower);
                to->si_upper = ptr_to_compat(from->si_upper);
                break;
        case SIL_FAULT_PKUERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_pkey = from->si_pkey;
                break;
        case SIL_FAULT_PERF_EVENT:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_perf_data = from->si_perf_data;
                to->si_perf_type = from->si_perf_type;
                to->si_perf_flags = from->si_perf_flags;
                break;
        case SIL_CHLD:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_status = from->si_status;
                to->si_utime = from->si_utime;
                to->si_stime = from->si_stime;
                break;
        case SIL_RT:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_int = from->si_int;
                break;
        case SIL_SYS:
                to->si_call_addr = ptr_to_compat(from->si_call_addr);
                to->si_syscall   = from->si_syscall;
                to->si_arch      = from->si_arch;
                break;
        }
}

int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
                           const struct kernel_siginfo *from)
{
        struct compat_siginfo new;

        copy_siginfo_to_external32(&new, from);
        if (copy_to_user(to, &new, sizeof(struct compat_siginfo)))
                return -EFAULT;
        return 0;
}

static int post_copy_siginfo_from_user32(kernel_siginfo_t *to,
                                         const struct compat_siginfo *from)
{
        clear_siginfo(to);
        to->si_signo = from->si_signo;
        to->si_errno = from->si_errno;
        to->si_code  = from->si_code;
        switch(siginfo_layout(from->si_signo, from->si_code)) {
        case SIL_KILL:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                break;
        case SIL_TIMER:
                to->si_tid     = from->si_tid;
                to->si_overrun = from->si_overrun;
                to->si_int     = from->si_int;
                break;
        case SIL_POLL:
                to->si_band = from->si_band;
                to->si_fd   = from->si_fd;
                break;
        case SIL_FAULT:
                to->si_addr = compat_ptr(from->si_addr);
                break;
        case SIL_FAULT_TRAPNO:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_trapno = from->si_trapno;
                break;
        case SIL_FAULT_MCEERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_addr_lsb = from->si_addr_lsb;
                break;
        case SIL_FAULT_BNDERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_lower = compat_ptr(from->si_lower);
                to->si_upper = compat_ptr(from->si_upper);
                break;
        case SIL_FAULT_PKUERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_pkey = from->si_pkey;
                break;
        case SIL_FAULT_PERF_EVENT:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_perf_data = from->si_perf_data;
                to->si_perf_type = from->si_perf_type;
                to->si_perf_flags = from->si_perf_flags;
                break;
        case SIL_CHLD:
                to->si_pid    = from->si_pid;
                to->si_uid    = from->si_uid;
                to->si_status = from->si_status;
#ifdef CONFIG_X86_X32_ABI
                if (in_x32_syscall()) {
                        to->si_utime = from->_sifields._sigchld_x32._utime;
                        to->si_stime = from->_sifields._sigchld_x32._stime;
                } else
#endif
                {
                        to->si_utime = from->si_utime;
                        to->si_stime = from->si_stime;
                }
                break;
        case SIL_RT:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_int = from->si_int;
                break;
        case SIL_SYS:
                to->si_call_addr = compat_ptr(from->si_call_addr);
                to->si_syscall   = from->si_syscall;
                to->si_arch      = from->si_arch;
                break;
        }
        return 0;
}

static int __copy_siginfo_from_user32(int signo, struct kernel_siginfo *to,
                                      const struct compat_siginfo __user *ufrom)
{
        struct compat_siginfo from;

        if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
                return -EFAULT;

        from.si_signo = signo;
        return post_copy_siginfo_from_user32(to, &from);
}

int copy_siginfo_from_user32(struct kernel_siginfo *to,
                             const struct compat_siginfo __user *ufrom)
{
        struct compat_siginfo from;

        if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
                return -EFAULT;

        return post_copy_siginfo_from_user32(to, &from);
}
#endif /* CONFIG_COMPAT */

/**
 *  do_sigtimedwait - wait for queued signals specified in @which
 *  @which: queued signals to wait for
 *  @info: if non-null, the signal's siginfo is returned here
 *  @ts: upper bound on process time suspension
 */
static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info,
                    const struct timespec64 *ts)
{
        ktime_t *to = NULL, timeout = KTIME_MAX;
        struct task_struct *tsk = current;
        sigset_t mask = *which;
        enum pid_type type;
        int sig, ret = 0;

        if (ts) {
                if (!timespec64_valid(ts))
                        return -EINVAL;
                timeout = timespec64_to_ktime(*ts);
                to = &timeout;
        }

        /*
         * Invert the set of allowed signals to get those we want to block.
         */
        sigdelsetmask(&mask, sigmask(SIGKILL) | sigmask(SIGSTOP));
        signotset(&mask);

        spin_lock_irq(&tsk->sighand->siglock);
        sig = dequeue_signal(&mask, info, &type);
        if (!sig && timeout) {
                /*
                 * None ready, temporarily unblock those we're interested
                 * while we are sleeping in so that we'll be awakened when
                 * they arrive. Unblocking is always fine, we can avoid
                 * set_current_blocked().
                 */
                tsk->real_blocked = tsk->blocked;
                sigandsets(&tsk->blocked, &tsk->blocked, &mask);
                recalc_sigpending();
                spin_unlock_irq(&tsk->sighand->siglock);

                __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
                ret = schedule_hrtimeout_range(to, tsk->timer_slack_ns,
                                               HRTIMER_MODE_REL);
                spin_lock_irq(&tsk->sighand->siglock);
                __set_task_blocked(tsk, &tsk->real_blocked);
                sigemptyset(&tsk->real_blocked);
                sig = dequeue_signal(&mask, info, &type);
        }
        spin_unlock_irq(&tsk->sighand->siglock);

        if (sig)
                return sig;
        return ret ? -EINTR : -EAGAIN;
}

/**
 *  sys_rt_sigtimedwait - synchronously wait for queued signals specified
 *                        in @uthese
 *  @uthese: queued signals to wait for
 *  @uinfo: if non-null, the signal's siginfo is returned here
 *  @uts: upper bound on process time suspension
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigtimedwait, const sigset_t __user *, uthese,
                siginfo_t __user *, uinfo,
                const struct __kernel_timespec __user *, uts,
                size_t, sigsetsize)
{
        sigset_t these;
        struct timespec64 ts;
        kernel_siginfo_t info;
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&these, uthese, sizeof(these)))
                return -EFAULT;

        if (uts) {
                if (get_timespec64(&ts, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}

#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(rt_sigtimedwait_time32, const sigset_t __user *, uthese,
                siginfo_t __user *, uinfo,
                const struct old_timespec32 __user *, uts,
                size_t, sigsetsize)
{
        sigset_t these;
        struct timespec64 ts;
        kernel_siginfo_t info;
        int ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&these, uthese, sizeof(these)))
                return -EFAULT;

        if (uts) {
                if (get_old_timespec32(&ts, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}
#endif

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time64, compat_sigset_t __user *, uthese,
                struct compat_siginfo __user *, uinfo,
                struct __kernel_timespec __user *, uts, compat_size_t, sigsetsize)
{
        sigset_t s;
        struct timespec64 t;
        kernel_siginfo_t info;
        long ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&s, uthese))
                return -EFAULT;

        if (uts) {
                if (get_timespec64(&t, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user32(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}

#ifdef CONFIG_COMPAT_32BIT_TIME
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time32, compat_sigset_t __user *, uthese,
                struct compat_siginfo __user *, uinfo,
                struct old_timespec32 __user *, uts, compat_size_t, sigsetsize)
{
        sigset_t s;
        struct timespec64 t;
        kernel_siginfo_t info;
        long ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&s, uthese))
                return -EFAULT;

        if (uts) {
                if (get_old_timespec32(&t, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user32(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}
#endif
#endif

static void prepare_kill_siginfo(int sig, struct kernel_siginfo *info,
                                 enum pid_type type)
{
        clear_siginfo(info);
        info->si_signo = sig;
        info->si_errno = 0;
        info->si_code = (type == PIDTYPE_PID) ? SI_TKILL : SI_USER;
        info->si_pid = task_tgid_vnr(current);
        info->si_uid = from_kuid_munged(current_user_ns(), current_uid());
}

/**
 *  sys_kill - send a signal to a process
 *  @pid: the PID of the process
 *  @sig: signal to be sent
 */
SYSCALL_DEFINE2(kill, pid_t, pid, int, sig)
{
        struct kernel_siginfo info;

        prepare_kill_siginfo(sig, &info, PIDTYPE_TGID);

        return kill_something_info(sig, &info, pid);
}

/*
 * Verify that the signaler and signalee either are in the same pid namespace
 * or that the signaler's pid namespace is an ancestor of the signalee's pid
 * namespace.
 */
static bool access_pidfd_pidns(struct pid *pid)
{
        struct pid_namespace *active = task_active_pid_ns(current);
        struct pid_namespace *p = ns_of_pid(pid);

        for (;;) {
                if (!p)
                        return false;
                if (p == active)
                        break;
                p = p->parent;
        }

        return true;
}

static int copy_siginfo_from_user_any(kernel_siginfo_t *kinfo,
                siginfo_t __user *info)
{
#ifdef CONFIG_COMPAT
        /*
         * Avoid hooking up compat syscalls and instead handle necessary
         * conversions here. Note, this is a stop-gap measure and should not be
         * considered a generic solution.
         */
        if (in_compat_syscall())
                return copy_siginfo_from_user32(
                        kinfo, (struct compat_siginfo __user *)info);
#endif
        return copy_siginfo_from_user(kinfo, info);
}

static struct pid *pidfd_to_pid(const struct file *file)
{
        struct pid *pid;

        pid = pidfd_pid(file);
        if (!IS_ERR(pid))
                return pid;

        return tgid_pidfd_to_pid(file);
}

#define PIDFD_SEND_SIGNAL_FLAGS                            \
        (PIDFD_SIGNAL_THREAD | PIDFD_SIGNAL_THREAD_GROUP | \
         PIDFD_SIGNAL_PROCESS_GROUP)

static int do_pidfd_send_signal(struct pid *pid, int sig, enum pid_type type,
                                siginfo_t __user *info, unsigned int flags)
{
        kernel_siginfo_t kinfo;

        switch (flags) {
        case PIDFD_SIGNAL_THREAD:
                type = PIDTYPE_PID;
                break;
        case PIDFD_SIGNAL_THREAD_GROUP:
                type = PIDTYPE_TGID;
                break;
        case PIDFD_SIGNAL_PROCESS_GROUP:
                type = PIDTYPE_PGID;
                break;
        }

        if (info) {
                int ret;

                ret = copy_siginfo_from_user_any(&kinfo, info);
                if (unlikely(ret))
                        return ret;

                if (unlikely(sig != kinfo.si_signo))
                        return -EINVAL;

                /* Only allow sending arbitrary signals to yourself. */
                if ((task_pid(current) != pid || type > PIDTYPE_TGID) &&
                    (kinfo.si_code >= 0 || kinfo.si_code == SI_TKILL))
                        return -EPERM;
        } else {
                prepare_kill_siginfo(sig, &kinfo, type);
        }

        if (type == PIDTYPE_PGID)
                return kill_pgrp_info(sig, &kinfo, pid);

        return kill_pid_info_type(sig, &kinfo, pid, type);
}

/**
 * sys_pidfd_send_signal - Signal a process through a pidfd
 * @pidfd:  file descriptor of the process
 * @sig:    signal to send
 * @info:   signal info
 * @flags:  future flags
 *
 * Send the signal to the thread group or to the individual thread depending
 * on PIDFD_THREAD.
 * In the future extension to @flags may be used to override the default scope
 * of @pidfd.
 *
 * Return: 0 on success, negative errno on failure
 */
SYSCALL_DEFINE4(pidfd_send_signal, int, pidfd, int, sig,
                siginfo_t __user *, info, unsigned int, flags)
{
        struct pid *pid;
        enum pid_type type;

        /* Enforce flags be set to 0 until we add an extension. */
        if (flags & ~PIDFD_SEND_SIGNAL_FLAGS)
                return -EINVAL;

        /* Ensure that only a single signal scope determining flag is set. */
        if (hweight32(flags & PIDFD_SEND_SIGNAL_FLAGS) > 1)
                return -EINVAL;

        switch (pidfd) {
        case PIDFD_SELF_THREAD:
                pid = get_task_pid(current, PIDTYPE_PID);
                type = PIDTYPE_PID;
                break;
        case PIDFD_SELF_THREAD_GROUP:
                pid = get_task_pid(current, PIDTYPE_TGID);
                type = PIDTYPE_TGID;
                break;
        default: {
                CLASS(fd, f)(pidfd);
                if (fd_empty(f))
                        return -EBADF;

                /* Is this a pidfd? */
                pid = pidfd_to_pid(fd_file(f));
                if (IS_ERR(pid))
                        return PTR_ERR(pid);

                if (!access_pidfd_pidns(pid))
                        return -EINVAL;

                /* Infer scope from the type of pidfd. */
                if (fd_file(f)->f_flags & PIDFD_THREAD)
                        type = PIDTYPE_PID;
                else
                        type = PIDTYPE_TGID;

                return do_pidfd_send_signal(pid, sig, type, info, flags);
        }
        }

        return do_pidfd_send_signal(pid, sig, type, info, flags);
}

static int
do_send_specific(pid_t tgid, pid_t pid, int sig, struct kernel_siginfo *info)
{
        struct task_struct *p;
        int error = -ESRCH;

        rcu_read_lock();
        p = find_task_by_vpid(pid);
        if (p && (tgid <= 0 || task_tgid_vnr(p) == tgid)) {
                error = check_kill_permission(sig, info, p);
                /*
                 * The null signal is a permissions and process existence
                 * probe.  No signal is actually delivered.
                 */
                if (!error && sig) {
                        error = do_send_sig_info(sig, info, p, PIDTYPE_PID);
                        /*
                         * If lock_task_sighand() failed we pretend the task
                         * dies after receiving the signal. The window is tiny,
                         * and the signal is private anyway.
                         */
                        if (unlikely(error == -ESRCH))
                                error = 0;
                }
        }
        rcu_read_unlock();

        return error;
}

static int do_tkill(pid_t tgid, pid_t pid, int sig)
{
        struct kernel_siginfo info;

        prepare_kill_siginfo(sig, &info, PIDTYPE_PID);

        return do_send_specific(tgid, pid, sig, &info);
}

/**
 *  sys_tgkill - send signal to one specific thread
 *  @tgid: the thread group ID of the thread
 *  @pid: the PID of the thread
 *  @sig: signal to be sent
 *
 *  This syscall also checks the @tgid and returns -ESRCH even if the PID
 *  exists but it's not belonging to the target process anymore. This
 *  method solves the problem of threads exiting and PIDs getting reused.
 */
SYSCALL_DEFINE3(tgkill, pid_t, tgid, pid_t, pid, int, sig)
{
        /* This is only valid for single tasks */
        if (pid <= 0 || tgid <= 0)
                return -EINVAL;

        return do_tkill(tgid, pid, sig);
}

/**
 *  sys_tkill - send signal to one specific task
 *  @pid: the PID of the task
 *  @sig: signal to be sent
 *
 *  Send a signal to only one task, even if it's a CLONE_THREAD task.
 */
SYSCALL_DEFINE2(tkill, pid_t, pid, int, sig)
{
        /* This is only valid for single tasks */
        if (pid <= 0)
                return -EINVAL;

        return do_tkill(0, pid, sig);
}

static int do_rt_sigqueueinfo(pid_t pid, int sig, kernel_siginfo_t *info)
{
        /* Not even root can pretend to send signals from the kernel.
         * Nor can they impersonate a kill()/tgkill(), which adds source info.
         */
        if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
            (task_pid_vnr(current) != pid))
                return -EPERM;

        /* POSIX.1b doesn't mention process groups.  */
        return kill_proc_info(sig, info, pid);
}

/**
 *  sys_rt_sigqueueinfo - send signal information to a signal
 *  @pid: the PID of the thread
 *  @sig: signal to be sent
 *  @uinfo: signal info to be sent
 */
SYSCALL_DEFINE3(rt_sigqueueinfo, pid_t, pid, int, sig,
                siginfo_t __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_sigqueueinfo(pid, sig, &info);
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(rt_sigqueueinfo,
                        compat_pid_t, pid,
                        int, sig,
                        struct compat_siginfo __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_sigqueueinfo(pid, sig, &info);
}
#endif

static int do_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, kernel_siginfo_t *info)
{
        /* This is only valid for single tasks */
        if (pid <= 0 || tgid <= 0)
                return -EINVAL;

        /* Not even root can pretend to send signals from the kernel.
         * Nor can they impersonate a kill()/tgkill(), which adds source info.
         */
        if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
            (task_pid_vnr(current) != pid))
                return -EPERM;

        return do_send_specific(tgid, pid, sig, info);
}

SYSCALL_DEFINE4(rt_tgsigqueueinfo, pid_t, tgid, pid_t, pid, int, sig,
                siginfo_t __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_tgsigqueueinfo,
                        compat_pid_t, tgid,
                        compat_pid_t, pid,
                        int, sig,
                        struct compat_siginfo __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}
#endif

/*
 * For kthreads only, must not be used if cloned with CLONE_SIGHAND
 */
void kernel_sigaction(int sig, __sighandler_t action)
{
        spin_lock_irq(&current->sighand->siglock);
        current->sighand->action[sig - 1].sa.sa_handler = action;
        if (action == SIG_IGN) {
                sigset_t mask;

                sigemptyset(&mask);
                sigaddset(&mask, sig);

                flush_sigqueue_mask(current, &mask, &current->signal->shared_pending);
                flush_sigqueue_mask(current, &mask, &current->pending);
                recalc_sigpending();
        }
        spin_unlock_irq(&current->sighand->siglock);
}
EXPORT_SYMBOL(kernel_sigaction);

void __weak sigaction_compat_abi(struct k_sigaction *act,
                struct k_sigaction *oact)
{
}

int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
{
        struct task_struct *p = current, *t;
        struct k_sigaction *k;
        sigset_t mask;

        if (!valid_signal(sig) || sig < 1 || (act && sig_kernel_only(sig)))
                return -EINVAL;

        k = &p->sighand->action[sig-1];

        spin_lock_irq(&p->sighand->siglock);
        if (k->sa.sa_flags & SA_IMMUTABLE) {
                spin_unlock_irq(&p->sighand->siglock);
                return -EINVAL;
        }
        if (oact)
                *oact = *k;

        /*
         * Make sure that we never accidentally claim to support SA_UNSUPPORTED,
         * e.g. by having an architecture use the bit in their uapi.
         */
        BUILD_BUG_ON(UAPI_SA_FLAGS & SA_UNSUPPORTED);

        /*
         * Clear unknown flag bits in order to allow userspace to detect missing
         * support for flag bits and to allow the kernel to use non-uapi bits
         * internally.
         */
        if (act)
                act->sa.sa_flags &= UAPI_SA_FLAGS;
        if (oact)
                oact->sa.sa_flags &= UAPI_SA_FLAGS;

        sigaction_compat_abi(act, oact);

        if (act) {
                bool was_ignored = k->sa.sa_handler == SIG_IGN;

                sigdelsetmask(&act->sa.sa_mask,
                              sigmask(SIGKILL) | sigmask(SIGSTOP));
                *k = *act;
                /*
                 * POSIX 3.3.1.3:
                 *  "Setting a signal action to SIG_IGN for a signal that is
                 *   pending shall cause the pending signal to be discarded,
                 *   whether or not it is blocked."
                 *
                 *  "Setting a signal action to SIG_DFL for a signal that is
                 *   pending and whose default action is to ignore the signal
                 *   (for example, SIGCHLD), shall cause the pending signal to
                 *   be discarded, whether or not it is blocked"
                 */
                if (sig_handler_ignored(sig_handler(p, sig), sig)) {
                        sigemptyset(&mask);
                        sigaddset(&mask, sig);
                        flush_sigqueue_mask(p, &mask, &p->signal->shared_pending);
                        for_each_thread(p, t)
                                flush_sigqueue_mask(p, &mask, &t->pending);
                } else if (was_ignored) {
                        posixtimer_sig_unignore(p, sig);
                }
        }

        spin_unlock_irq(&p->sighand->siglock);
        return 0;
}

#ifdef CONFIG_DYNAMIC_SIGFRAME
static inline void sigaltstack_lock(void)
        __acquires(&current->sighand->siglock)
{
        spin_lock_irq(&current->sighand->siglock);
}

static inline void sigaltstack_unlock(void)
        __releases(&current->sighand->siglock)
{
        spin_unlock_irq(&current->sighand->siglock);
}
#else
static inline void sigaltstack_lock(void) { }
static inline void sigaltstack_unlock(void) { }
#endif

static int
do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp,
                size_t min_ss_size)
{
        struct task_struct *t = current;
        int ret = 0;

        if (oss) {
                memset(oss, 0, sizeof(stack_t));
                oss->ss_sp = (void __user *) t->sas_ss_sp;
                oss->ss_size = t->sas_ss_size;
                oss->ss_flags = sas_ss_flags(sp) |
                        (current->sas_ss_flags & SS_FLAG_BITS);
        }

        if (ss) {
                void __user *ss_sp = ss->ss_sp;
                size_t ss_size = ss->ss_size;
                unsigned ss_flags = ss->ss_flags;
                int ss_mode;

                if (unlikely(on_sig_stack(sp)))
                        return -EPERM;

                ss_mode = ss_flags & ~SS_FLAG_BITS;
                if (unlikely(ss_mode != SS_DISABLE && ss_mode != SS_ONSTACK &&
                                ss_mode != 0))
                        return -EINVAL;

                /*
                 * Return before taking any locks if no actual
                 * sigaltstack changes were requested.
                 */
                if (t->sas_ss_sp == (unsigned long)ss_sp &&
                    t->sas_ss_size == ss_size &&
                    t->sas_ss_flags == ss_flags)
                        return 0;

                sigaltstack_lock();
                if (ss_mode == SS_DISABLE) {
                        ss_size = 0;
                        ss_sp = NULL;
                } else {
                        if (unlikely(ss_size < min_ss_size))
                                ret = -ENOMEM;
                        if (!sigaltstack_size_valid(ss_size))
                                ret = -ENOMEM;
                }
                if (!ret) {
                        t->sas_ss_sp = (unsigned long) ss_sp;
                        t->sas_ss_size = ss_size;
                        t->sas_ss_flags = ss_flags;
                }
                sigaltstack_unlock();
        }
        return ret;
}

SYSCALL_DEFINE2(sigaltstack,const stack_t __user *,uss, stack_t __user *,uoss)
{
        stack_t new, old;
        int err;
        if (uss && copy_from_user(&new, uss, sizeof(stack_t)))
                return -EFAULT;
        err = do_sigaltstack(uss ? &new : NULL, uoss ? &old : NULL,
                              current_user_stack_pointer(),
                              MINSIGSTKSZ);
        if (!err && uoss && copy_to_user(uoss, &old, sizeof(stack_t)))
                err = -EFAULT;
        return err;
}

int restore_altstack(const stack_t __user *uss)
{
        stack_t new;
        if (copy_from_user(&new, uss, sizeof(stack_t)))
                return -EFAULT;
        (void)do_sigaltstack(&new, NULL, current_user_stack_pointer(),
                             MINSIGSTKSZ);
        /* squash all but EFAULT for now */
        return 0;
}

int __save_altstack(stack_t __user *uss, unsigned long sp)
{
        struct task_struct *t = current;
        int err = __put_user((void __user *)t->sas_ss_sp, &uss->ss_sp) |
                __put_user(t->sas_ss_flags, &uss->ss_flags) |
                __put_user(t->sas_ss_size, &uss->ss_size);
        return err;
}

#ifdef CONFIG_COMPAT
static int do_compat_sigaltstack(const compat_stack_t __user *uss_ptr,
                                 compat_stack_t __user *uoss_ptr)
{
        stack_t uss, uoss;
        int ret;

        if (uss_ptr) {
                compat_stack_t uss32;
                if (copy_from_user(&uss32, uss_ptr, sizeof(compat_stack_t)))
                        return -EFAULT;
                uss.ss_sp = compat_ptr(uss32.ss_sp);
                uss.ss_flags = uss32.ss_flags;
                uss.ss_size = uss32.ss_size;
        }
        ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss,
                             compat_user_stack_pointer(),
                             COMPAT_MINSIGSTKSZ);
        if (ret >= 0 && uoss_ptr)  {
                compat_stack_t old;
                memset(&old, 0, sizeof(old));
                old.ss_sp = ptr_to_compat(uoss.ss_sp);
                old.ss_flags = uoss.ss_flags;
                old.ss_size = uoss.ss_size;
                if (copy_to_user(uoss_ptr, &old, sizeof(compat_stack_t)))
                        ret = -EFAULT;
        }
        return ret;
}

COMPAT_SYSCALL_DEFINE2(sigaltstack,
                        const compat_stack_t __user *, uss_ptr,
                        compat_stack_t __user *, uoss_ptr)
{
        return do_compat_sigaltstack(uss_ptr, uoss_ptr);
}

int compat_restore_altstack(const compat_stack_t __user *uss)
{
        int err = do_compat_sigaltstack(uss, NULL);
        /* squash all but -EFAULT for now */
        return err == -EFAULT ? err : 0;
}

int __compat_save_altstack(compat_stack_t __user *uss, unsigned long sp)
{
        int err;
        struct task_struct *t = current;
        err = __put_user(ptr_to_compat((void __user *)t->sas_ss_sp),
                         &uss->ss_sp) |
                __put_user(t->sas_ss_flags, &uss->ss_flags) |
                __put_user(t->sas_ss_size, &uss->ss_size);
        return err;
}
#endif

#ifdef __ARCH_WANT_SYS_SIGPENDING

/**
 *  sys_sigpending - examine pending signals
 *  @uset: where mask of pending signal is returned
 */
SYSCALL_DEFINE1(sigpending, old_sigset_t __user *, uset)
{
        sigset_t set;

        if (sizeof(old_sigset_t) > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        if (copy_to_user(uset, &set, sizeof(old_sigset_t)))
                return -EFAULT;

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE1(sigpending, compat_old_sigset_t __user *, set32)
{
        sigset_t set;

        do_sigpending(&set);

        return put_user(set.sig[0], set32);
}
#endif

#endif

#ifdef __ARCH_WANT_SYS_SIGPROCMASK
/**
 *  sys_sigprocmask - examine and change blocked signals
 *  @how: whether to add, remove, or set signals
 *  @nset: signals to add or remove (if non-null)
 *  @oset: previous value of signal mask if non-null
 *
 * Some platforms have their own version with special arguments;
 * others support only sys_rt_sigprocmask.
 */

SYSCALL_DEFINE3(sigprocmask, int, how, old_sigset_t __user *, nset,
                old_sigset_t __user *, oset)
{
        old_sigset_t old_set, new_set;
        sigset_t new_blocked;

        old_set = current->blocked.sig[0];

        if (nset) {
                if (copy_from_user(&new_set, nset, sizeof(*nset)))
                        return -EFAULT;

                new_blocked = current->blocked;

                switch (how) {
                case SIG_BLOCK:
                        sigaddsetmask(&new_blocked, new_set);
                        break;
                case SIG_UNBLOCK:
                        sigdelsetmask(&new_blocked, new_set);
                        break;
                case SIG_SETMASK:
                        new_blocked.sig[0] = new_set;
                        break;
                default:
                        return -EINVAL;
                }

                set_current_blocked(&new_blocked);
        }

        if (oset) {
                if (copy_to_user(oset, &old_set, sizeof(*oset)))
                        return -EFAULT;
        }

        return 0;
}
#endif /* __ARCH_WANT_SYS_SIGPROCMASK */

#ifndef CONFIG_ODD_RT_SIGACTION
/**
 *  sys_rt_sigaction - alter an action taken by a process
 *  @sig: signal to be sent
 *  @act: new sigaction
 *  @oact: used to save the previous sigaction
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigaction, int, sig,
                const struct sigaction __user *, act,
                struct sigaction __user *, oact,
                size_t, sigsetsize)
{
        struct k_sigaction new_sa, old_sa;
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (act && copy_from_user(&new_sa.sa, act, sizeof(new_sa.sa)))
                return -EFAULT;

        ret = do_sigaction(sig, act ? &new_sa : NULL, oact ? &old_sa : NULL);
        if (ret)
                return ret;

        if (oact && copy_to_user(oact, &old_sa.sa, sizeof(old_sa.sa)))
                return -EFAULT;

        return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigaction, int, sig,
                const struct compat_sigaction __user *, act,
                struct compat_sigaction __user *, oact,
                compat_size_t, sigsetsize)
{
        struct k_sigaction new_ka, old_ka;
#ifdef __ARCH_HAS_SA_RESTORER
        compat_uptr_t restorer;
#endif
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(compat_sigset_t))
                return -EINVAL;

        if (act) {
                compat_uptr_t handler;
                ret = get_user(handler, &act->sa_handler);
                new_ka.sa.sa_handler = compat_ptr(handler);
#ifdef __ARCH_HAS_SA_RESTORER
                ret |= get_user(restorer, &act->sa_restorer);
                new_ka.sa.sa_restorer = compat_ptr(restorer);
#endif
                ret |= get_compat_sigset(&new_ka.sa.sa_mask, &act->sa_mask);
                ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags);
                if (ret)
                        return -EFAULT;
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
        if (!ret && oact) {
                ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), 
                               &oact->sa_handler);
                ret |= put_compat_sigset(&oact->sa_mask, &old_ka.sa.sa_mask,
                                         sizeof(oact->sa_mask));
                ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags);
#ifdef __ARCH_HAS_SA_RESTORER
                ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer),
                                &oact->sa_restorer);
#endif
        }
        return ret;
}
#endif
#endif /* !CONFIG_ODD_RT_SIGACTION */

#ifdef CONFIG_OLD_SIGACTION
SYSCALL_DEFINE3(sigaction, int, sig,
                const struct old_sigaction __user *, act,
                struct old_sigaction __user *, oact)
{
        struct k_sigaction new_ka, old_ka;
        int ret;

        if (act) {
                old_sigset_t mask;
                if (!access_ok(act, sizeof(*act)) ||
                    __get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
                    __get_user(new_ka.sa.sa_restorer, &act->sa_restorer) ||
                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
                    __get_user(mask, &act->sa_mask))
                        return -EFAULT;
#ifdef __ARCH_HAS_KA_RESTORER
                new_ka.ka_restorer = NULL;
#endif
                siginitset(&new_ka.sa.sa_mask, mask);
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);

        if (!ret && oact) {
                if (!access_ok(oact, sizeof(*oact)) ||
                    __put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
                    __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) ||
                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
                        return -EFAULT;
        }

        return ret;
}
#endif
#ifdef CONFIG_COMPAT_OLD_SIGACTION
COMPAT_SYSCALL_DEFINE3(sigaction, int, sig,
                const struct compat_old_sigaction __user *, act,
                struct compat_old_sigaction __user *, oact)
{
        struct k_sigaction new_ka, old_ka;
        int ret;
        compat_old_sigset_t mask;
        compat_uptr_t handler, restorer;

        if (act) {
                if (!access_ok(act, sizeof(*act)) ||
                    __get_user(handler, &act->sa_handler) ||
                    __get_user(restorer, &act->sa_restorer) ||
                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
                    __get_user(mask, &act->sa_mask))
                        return -EFAULT;

#ifdef __ARCH_HAS_KA_RESTORER
                new_ka.ka_restorer = NULL;
#endif
                new_ka.sa.sa_handler = compat_ptr(handler);
                new_ka.sa.sa_restorer = compat_ptr(restorer);
                siginitset(&new_ka.sa.sa_mask, mask);
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);

        if (!ret && oact) {
                if (!access_ok(oact, sizeof(*oact)) ||
                    __put_user(ptr_to_compat(old_ka.sa.sa_handler),
                               &oact->sa_handler) ||
                    __put_user(ptr_to_compat(old_ka.sa.sa_restorer),
                               &oact->sa_restorer) ||
                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
                        return -EFAULT;
        }
        return ret;
}
#endif

#ifdef CONFIG_SGETMASK_SYSCALL

/*
 * For backwards compatibility.  Functionality superseded by sigprocmask.
 */
SYSCALL_DEFINE0(sgetmask)
{
        /* SMP safe */
        return current->blocked.sig[0];
}

SYSCALL_DEFINE1(ssetmask, int, newmask)
{
        int old = current->blocked.sig[0];
        sigset_t newset;

        siginitset(&newset, newmask);
        set_current_blocked(&newset);

        return old;
}
#endif /* CONFIG_SGETMASK_SYSCALL */

#ifdef __ARCH_WANT_SYS_SIGNAL
/*
 * For backwards compatibility.  Functionality superseded by sigaction.
 */
SYSCALL_DEFINE2(signal, int, sig, __sighandler_t, handler)
{
        struct k_sigaction new_sa, old_sa;
        int ret;

        new_sa.sa.sa_handler = handler;
        new_sa.sa.sa_flags = SA_ONESHOT | SA_NOMASK;
        sigemptyset(&new_sa.sa.sa_mask);

        ret = do_sigaction(sig, &new_sa, &old_sa);

        return ret ? ret : (unsigned long)old_sa.sa.sa_handler;
}
#endif /* __ARCH_WANT_SYS_SIGNAL */

#ifdef __ARCH_WANT_SYS_PAUSE

SYSCALL_DEFINE0(pause)
{
        while (!signal_pending(current)) {
                __set_current_state(TASK_INTERRUPTIBLE);
                schedule();
        }
        return -ERESTARTNOHAND;
}

#endif

static int sigsuspend(sigset_t *set)
{
        current->saved_sigmask = current->blocked;
        set_current_blocked(set);

        while (!signal_pending(current)) {
                __set_current_state(TASK_INTERRUPTIBLE);
                schedule();
        }
        set_restore_sigmask();
        return -ERESTARTNOHAND;
}

/**
 *  sys_rt_sigsuspend - replace the signal mask for a value with the
 *        @unewset value until a signal is received
 *  @unewset: new signal mask value
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE2(rt_sigsuspend, sigset_t __user *, unewset, size_t, sigsetsize)
{
        sigset_t newset;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&newset, unewset, sizeof(newset)))
                return -EFAULT;
        return sigsuspend(&newset);
}
 
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigsuspend, compat_sigset_t __user *, unewset, compat_size_t, sigsetsize)
{
        sigset_t newset;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&newset, unewset))
                return -EFAULT;
        return sigsuspend(&newset);
}
#endif

#ifdef CONFIG_OLD_SIGSUSPEND
SYSCALL_DEFINE1(sigsuspend, old_sigset_t, mask)
{
        sigset_t blocked;
        siginitset(&blocked, mask);
        return sigsuspend(&blocked);
}
#endif
#ifdef CONFIG_OLD_SIGSUSPEND3
SYSCALL_DEFINE3(sigsuspend, int, unused1, int, unused2, old_sigset_t, mask)
{
        sigset_t blocked;
        siginitset(&blocked, mask);
        return sigsuspend(&blocked);
}
#endif

__weak const char *arch_vma_name(struct vm_area_struct *vma)
{
        return NULL;
}

static inline void siginfo_buildtime_checks(void)
{
        BUILD_BUG_ON(sizeof(struct siginfo) != SI_MAX_SIZE);

        /* Verify the offsets in the two siginfos match */
#define CHECK_OFFSET(field) \
        BUILD_BUG_ON(offsetof(siginfo_t, field) != offsetof(kernel_siginfo_t, field))

        /* kill */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);

        /* timer */
        CHECK_OFFSET(si_tid);
        CHECK_OFFSET(si_overrun);
        CHECK_OFFSET(si_value);

        /* rt */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);
        CHECK_OFFSET(si_value);

        /* sigchld */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);
        CHECK_OFFSET(si_status);
        CHECK_OFFSET(si_utime);
        CHECK_OFFSET(si_stime);

        /* sigfault */
        CHECK_OFFSET(si_addr);
        CHECK_OFFSET(si_trapno);
        CHECK_OFFSET(si_addr_lsb);
        CHECK_OFFSET(si_lower);
        CHECK_OFFSET(si_upper);
        CHECK_OFFSET(si_pkey);
        CHECK_OFFSET(si_perf_data);
        CHECK_OFFSET(si_perf_type);
        CHECK_OFFSET(si_perf_flags);

        /* sigpoll */
        CHECK_OFFSET(si_band);
        CHECK_OFFSET(si_fd);

        /* sigsys */
        CHECK_OFFSET(si_call_addr);
        CHECK_OFFSET(si_syscall);
        CHECK_OFFSET(si_arch);
#undef CHECK_OFFSET

        /* usb asyncio */
        BUILD_BUG_ON(offsetof(struct siginfo, si_pid) !=
                     offsetof(struct siginfo, si_addr));
        if (sizeof(int) == sizeof(void __user *)) {
                BUILD_BUG_ON(sizeof_field(struct siginfo, si_pid) !=
                             sizeof(void __user *));
        } else {
                BUILD_BUG_ON((sizeof_field(struct siginfo, si_pid) +
                              sizeof_field(struct siginfo, si_uid)) !=
                             sizeof(void __user *));
                BUILD_BUG_ON(offsetofend(struct siginfo, si_pid) !=
                             offsetof(struct siginfo, si_uid));
        }
#ifdef CONFIG_COMPAT
        BUILD_BUG_ON(offsetof(struct compat_siginfo, si_pid) !=
                     offsetof(struct compat_siginfo, si_addr));
        BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
                     sizeof(compat_uptr_t));
        BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
                     sizeof_field(struct siginfo, si_pid));
#endif
}

#if defined(CONFIG_SYSCTL)
static const struct ctl_table signal_debug_table[] = {
#ifdef CONFIG_SYSCTL_EXCEPTION_TRACE
        {
                .procname        = "exception-trace",
                .data                = &show_unhandled_signals,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
#endif
};

static const struct ctl_table signal_table[] = {
        {
                .procname        = "print-fatal-signals",
                .data                = &print_fatal_signals,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
};

static int __init init_signal_sysctls(void)
{
        register_sysctl_init("debug", signal_debug_table);
        register_sysctl_init("kernel", signal_table);
        return 0;
}
early_initcall(init_signal_sysctls);
#endif /* CONFIG_SYSCTL */

void __init signals_init(void)
{
        siginfo_buildtime_checks();

        sigqueue_cachep = KMEM_CACHE(sigqueue, SLAB_PANIC | SLAB_ACCOUNT);
}

#ifdef CONFIG_KGDB_KDB
#include <linux/kdb.h>
/*
 * kdb_send_sig - Allows kdb to send signals without exposing
 * signal internals.  This function checks if the required locks are
 * available before calling the main signal code, to avoid kdb
 * deadlocks.
 */
void kdb_send_sig(struct task_struct *t, int sig)
{
        static struct task_struct *kdb_prev_t;
        int new_t, ret;
        if (!spin_trylock(&t->sighand->siglock)) {
                kdb_printf("Can't do kill command now.\n"
                           "The sigmask lock is held somewhere else in "
                           "kernel, try again later\n");
                return;
        }
        new_t = kdb_prev_t != t;
        kdb_prev_t = t;
        if (!task_is_running(t) && new_t) {
                spin_unlock(&t->sighand->siglock);
                kdb_printf("Process is not RUNNING, sending a signal from "
                           "kdb risks deadlock\n"
                           "on the run queue locks. "
                           "The signal has _not_ been sent.\n"
                           "Reissue the kill command if you want to risk "
                           "the deadlock.\n");
                return;
        }
        ret = send_signal_locked(sig, SEND_SIG_PRIV, t, PIDTYPE_PID);
        spin_unlock(&t->sighand->siglock);
        if (ret)
                kdb_printf("Fail to deliver Signal %d to process %d.\n",
                           sig, t->pid);
        else
                kdb_printf("Signal %d is sent to process %d.\n", sig, t->pid);
}
#endif        /* CONFIG_KGDB_KDB */






























































































































































































































































































  148 


















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Based on arch/arm/include/asm/ptrace.h
 *
 * Copyright (C) 1996-2003 Russell King
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_PTRACE_H
#define __ASM_PTRACE_H

#include <asm/cpufeature.h>

#include <uapi/asm/ptrace.h>

/* Current Exception Level values, as contained in CurrentEL */
#define CurrentEL_EL1                (1 << 2)
#define CurrentEL_EL2                (2 << 2)

#define INIT_PSTATE_EL1 \
        (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h)
#define INIT_PSTATE_EL2 \
        (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h)

#include <linux/irqchip/arm-gic-v3-prio.h>

#define GIC_PRIO_IRQON                GICV3_PRIO_UNMASKED
#define GIC_PRIO_IRQOFF                GICV3_PRIO_IRQ

#define GIC_PRIO_PSR_I_SET        GICV3_PRIO_PSR_I_SET

/* Additional SPSR bits not exposed in the UABI */
#define PSR_MODE_THREAD_BIT        (1 << 0)
#define PSR_IL_BIT                (1 << 20)

/* AArch32-specific ptrace requests */
#define COMPAT_PTRACE_GETREGS                12
#define COMPAT_PTRACE_SETREGS                13
#define COMPAT_PTRACE_GET_THREAD_AREA        22
#define COMPAT_PTRACE_SET_SYSCALL        23
#define COMPAT_PTRACE_GETVFPREGS        27
#define COMPAT_PTRACE_SETVFPREGS        28
#define COMPAT_PTRACE_GETHBPREGS        29
#define COMPAT_PTRACE_SETHBPREGS        30

/* SPSR_ELx bits for exceptions taken from AArch32 */
#define PSR_AA32_MODE_MASK        0x0000001f
#define PSR_AA32_MODE_USR        0x00000010
#define PSR_AA32_MODE_FIQ        0x00000011
#define PSR_AA32_MODE_IRQ        0x00000012
#define PSR_AA32_MODE_SVC        0x00000013
#define PSR_AA32_MODE_ABT        0x00000017
#define PSR_AA32_MODE_HYP        0x0000001a
#define PSR_AA32_MODE_UND        0x0000001b
#define PSR_AA32_MODE_SYS        0x0000001f
#define PSR_AA32_T_BIT                0x00000020
#define PSR_AA32_F_BIT                0x00000040
#define PSR_AA32_I_BIT                0x00000080
#define PSR_AA32_A_BIT                0x00000100
#define PSR_AA32_E_BIT                0x00000200
#define PSR_AA32_PAN_BIT        0x00400000
#define PSR_AA32_SSBS_BIT        0x00800000
#define PSR_AA32_DIT_BIT        0x01000000
#define PSR_AA32_Q_BIT                0x08000000
#define PSR_AA32_V_BIT                0x10000000
#define PSR_AA32_C_BIT                0x20000000
#define PSR_AA32_Z_BIT                0x40000000
#define PSR_AA32_N_BIT                0x80000000
#define PSR_AA32_IT_MASK        0x0600fc00        /* If-Then execution state mask */
#define PSR_AA32_GE_MASK        0x000f0000

#ifdef CONFIG_CPU_BIG_ENDIAN
#define PSR_AA32_ENDSTATE        PSR_AA32_E_BIT
#else
#define PSR_AA32_ENDSTATE        0
#endif

/* AArch32 CPSR bits, as seen in AArch32 */
#define COMPAT_PSR_DIT_BIT        0x00200000

/*
 * These are 'magic' values for PTRACE_PEEKUSR that return info about where a
 * process is located in memory.
 */
#define COMPAT_PT_TEXT_ADDR                0x10000
#define COMPAT_PT_DATA_ADDR                0x10004
#define COMPAT_PT_TEXT_END_ADDR                0x10008

/*
 * If pt_regs.syscallno == NO_SYSCALL, then the thread is not executing
 * a syscall -- i.e., its most recent entry into the kernel from
 * userspace was not via SVC, or otherwise a tracer cancelled the syscall.
 *
 * This must have the value -1, for ABI compatibility with ptrace etc.
 */
#define NO_SYSCALL (-1)

#ifndef __ASSEMBLY__
#include <linux/bug.h>
#include <linux/types.h>

#include <asm/stacktrace/frame.h>

/* sizeof(struct user) for AArch32 */
#define COMPAT_USER_SZ        296

/* Architecturally defined mapping between AArch32 and AArch64 registers */
#define compat_usr(x)        regs[(x)]
#define compat_fp        regs[11]
#define compat_sp        regs[13]
#define compat_lr        regs[14]
#define compat_sp_hyp        regs[15]
#define compat_lr_irq        regs[16]
#define compat_sp_irq        regs[17]
#define compat_lr_svc        regs[18]
#define compat_sp_svc        regs[19]
#define compat_lr_abt        regs[20]
#define compat_sp_abt        regs[21]
#define compat_lr_und        regs[22]
#define compat_sp_und        regs[23]
#define compat_r8_fiq        regs[24]
#define compat_r9_fiq        regs[25]
#define compat_r10_fiq        regs[26]
#define compat_r11_fiq        regs[27]
#define compat_r12_fiq        regs[28]
#define compat_sp_fiq        regs[29]
#define compat_lr_fiq        regs[30]

static inline unsigned long compat_psr_to_pstate(const unsigned long psr)
{
        unsigned long pstate;

        pstate = psr & ~COMPAT_PSR_DIT_BIT;

        if (psr & COMPAT_PSR_DIT_BIT)
                pstate |= PSR_AA32_DIT_BIT;

        return pstate;
}

static inline unsigned long pstate_to_compat_psr(const unsigned long pstate)
{
        unsigned long psr;

        psr = pstate & ~PSR_AA32_DIT_BIT;

        if (pstate & PSR_AA32_DIT_BIT)
                psr |= COMPAT_PSR_DIT_BIT;

        return psr;
}

/*
 * This struct defines the way the registers are stored on the stack during an
 * exception. struct user_pt_regs must form a prefix of struct pt_regs.
 */
struct pt_regs {
        union {
                struct user_pt_regs user_regs;
                struct {
                        u64 regs[31];
                        u64 sp;
                        u64 pc;
                        u64 pstate;
                };
        };
        u64 orig_x0;
        s32 syscallno;
        u32 pmr;

        u64 sdei_ttbr1;
        struct frame_record_meta stackframe;

        /* Only valid for some EL1 exceptions. */
        u64 lockdep_hardirqs;
        u64 exit_rcu;
};

/* For correct stack alignment, pt_regs has to be a multiple of 16 bytes. */
static_assert(IS_ALIGNED(sizeof(struct pt_regs), 16));

static inline bool in_syscall(struct pt_regs const *regs)
{
        return regs->syscallno != NO_SYSCALL;
}

static inline void forget_syscall(struct pt_regs *regs)
{
        regs->syscallno = NO_SYSCALL;
}

#define MAX_REG_OFFSET offsetof(struct pt_regs, pstate)

#define arch_has_single_step()        (1)

#ifdef CONFIG_COMPAT
#define compat_thumb_mode(regs) \
        (((regs)->pstate & PSR_AA32_T_BIT))
#else
#define compat_thumb_mode(regs) (0)
#endif

#define user_mode(regs)        \
        (((regs)->pstate & PSR_MODE_MASK) == PSR_MODE_EL0t)

#define compat_user_mode(regs)        \
        (((regs)->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) == \
         (PSR_MODE32_BIT | PSR_MODE_EL0t))

#define processor_mode(regs) \
        ((regs)->pstate & PSR_MODE_MASK)

#define irqs_priority_unmasked(regs)                                        \
        (system_uses_irq_prio_masking() ?                                \
                (regs)->pmr == GIC_PRIO_IRQON :                                \
                true)

#define interrupts_enabled(regs)                        \
        (!((regs)->pstate & PSR_I_BIT) && irqs_priority_unmasked(regs))

#define fast_interrupts_enabled(regs) \
        (!((regs)->pstate & PSR_F_BIT))

static inline unsigned long user_stack_pointer(struct pt_regs *regs)
{
        if (compat_user_mode(regs))
                return regs->compat_sp;
        return regs->sp;
}

extern int regs_query_register_offset(const char *name);
extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs,
                                               unsigned int n);

/**
 * regs_get_register() - get register value from its offset
 * @regs:        pt_regs from which register value is gotten
 * @offset:        offset of the register.
 *
 * regs_get_register returns the value of a register whose offset from @regs.
 * The @offset is the offset of the register in struct pt_regs.
 * If @offset is bigger than MAX_REG_OFFSET, this returns 0.
 */
static inline u64 regs_get_register(struct pt_regs *regs, unsigned int offset)
{
        u64 val = 0;

        WARN_ON(offset & 7);

        offset >>= 3;
        switch (offset) {
        case 0 ... 30:
                val = regs->regs[offset];
                break;
        case offsetof(struct pt_regs, sp) >> 3:
                val = regs->sp;
                break;
        case offsetof(struct pt_regs, pc) >> 3:
                val = regs->pc;
                break;
        case offsetof(struct pt_regs, pstate) >> 3:
                val = regs->pstate;
                break;
        default:
                val = 0;
        }

        return val;
}

/*
 * Read a register given an architectural register index r.
 * This handles the common case where 31 means XZR, not SP.
 */
static inline unsigned long pt_regs_read_reg(const struct pt_regs *regs, int r)
{
        return (r == 31) ? 0 : regs->regs[r];
}

/*
 * Write a register given an architectural register index r.
 * This handles the common case where 31 means XZR, not SP.
 */
static inline void pt_regs_write_reg(struct pt_regs *regs, int r,
                                     unsigned long val)
{
        if (r != 31)
                regs->regs[r] = val;
}

/* Valid only for Kernel mode traps. */
static inline unsigned long kernel_stack_pointer(struct pt_regs *regs)
{
        return regs->sp;
}

static inline unsigned long regs_return_value(struct pt_regs *regs)
{
        unsigned long val = regs->regs[0];

        /*
         * Audit currently uses regs_return_value() instead of
         * syscall_get_return_value(). Apply the same sign-extension here until
         * audit is updated to use syscall_get_return_value().
         */
        if (compat_user_mode(regs))
                val = sign_extend64(val, 31);

        return val;
}

static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
{
        regs->regs[0] = rc;
}

/**
 * regs_get_kernel_argument() - get Nth function argument in kernel
 * @regs:        pt_regs of that context
 * @n:                function argument number (start from 0)
 *
 * regs_get_argument() returns @n th argument of the function call.
 *
 * Note that this chooses the most likely register mapping. In very rare
 * cases this may not return correct data, for example, if one of the
 * function parameters is 16 bytes or bigger. In such cases, we cannot
 * get access the parameter correctly and the register assignment of
 * subsequent parameters will be shifted.
 */
static inline unsigned long regs_get_kernel_argument(struct pt_regs *regs,
                                                     unsigned int n)
{
#define NR_REG_ARGUMENTS 8
        if (n < NR_REG_ARGUMENTS)
                return pt_regs_read_reg(regs, n);
        return 0;
}

/* We must avoid circular header include via sched.h */
struct task_struct;
int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task);

static inline unsigned long instruction_pointer(struct pt_regs *regs)
{
        return regs->pc;
}
static inline void instruction_pointer_set(struct pt_regs *regs,
                unsigned long val)
{
        regs->pc = val;
}

static inline unsigned long frame_pointer(struct pt_regs *regs)
{
        return regs->regs[29];
}

#define procedure_link_pointer(regs)        ((regs)->regs[30])

static inline void procedure_link_pointer_set(struct pt_regs *regs,
                                           unsigned long val)
{
        procedure_link_pointer(regs) = val;
}

extern unsigned long profile_pc(struct pt_regs *regs);

#endif /* __ASSEMBLY__ */
#endif
























































































   10 




































    1 




    3 
    1 

    2 


    1 


    1 






  300 
  301 






   17 



   16 
    1 









    1 





















   84 











   84 



   73 
   10 

    1 


   14 

   70 




















   84 



















   84 

   84 









  530 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012,2013 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * Derived from arch/arm/kvm/reset.c
 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/hw_breakpoint.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/types.h>

#include <kvm/arm_arch_timer.h>

#include <asm/cpufeature.h>
#include <asm/cputype.h>
#include <asm/fpsimd.h>
#include <asm/ptrace.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nested.h>
#include <asm/virt.h>

/* Maximum phys_shift supported for any VM on this host */
static u32 __ro_after_init kvm_ipa_limit;
unsigned int __ro_after_init kvm_host_sve_max_vl;

/*
 * ARMv8 Reset Values
 */
#define VCPU_RESET_PSTATE_EL1        (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \
                                 PSR_F_BIT | PSR_D_BIT)

#define VCPU_RESET_PSTATE_EL2        (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \
                                 PSR_F_BIT | PSR_D_BIT)

#define VCPU_RESET_PSTATE_SVC        (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \
                                 PSR_AA32_I_BIT | PSR_AA32_F_BIT)

unsigned int __ro_after_init kvm_sve_max_vl;

int __init kvm_arm_init_sve(void)
{
        if (system_supports_sve()) {
                kvm_sve_max_vl = sve_max_virtualisable_vl();
                kvm_host_sve_max_vl = sve_max_vl();
                kvm_nvhe_sym(kvm_host_sve_max_vl) = kvm_host_sve_max_vl;

                /*
                 * The get_sve_reg()/set_sve_reg() ioctl interface will need
                 * to be extended with multiple register slice support in
                 * order to support vector lengths greater than
                 * VL_ARCH_MAX:
                 */
                if (WARN_ON(kvm_sve_max_vl > VL_ARCH_MAX))
                        kvm_sve_max_vl = VL_ARCH_MAX;

                /*
                 * Don't even try to make use of vector lengths that
                 * aren't available on all CPUs, for now:
                 */
                if (kvm_sve_max_vl < sve_max_vl())
                        pr_warn("KVM: SVE vector length for guests limited to %u bytes\n",
                                kvm_sve_max_vl);
        }

        return 0;
}

static void kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)
{
        vcpu->arch.sve_max_vl = kvm_sve_max_vl;

        /*
         * Userspace can still customize the vector lengths by writing
         * KVM_REG_ARM64_SVE_VLS.  Allocation is deferred until
         * kvm_arm_vcpu_finalize(), which freezes the configuration.
         */
        set_bit(KVM_ARCH_FLAG_GUEST_HAS_SVE, &vcpu->kvm->arch.flags);
}

/*
 * Finalize vcpu's maximum SVE vector length, allocating
 * vcpu->arch.sve_state as necessary.
 */
static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu)
{
        void *buf;
        unsigned int vl;
        size_t reg_sz;
        int ret;

        vl = vcpu->arch.sve_max_vl;

        /*
         * Responsibility for these properties is shared between
         * kvm_arm_init_sve(), kvm_vcpu_enable_sve() and
         * set_sve_vls().  Double-check here just to be sure:
         */
        if (WARN_ON(!sve_vl_valid(vl) || vl > sve_max_virtualisable_vl() ||
                    vl > VL_ARCH_MAX))
                return -EIO;

        reg_sz = vcpu_sve_state_size(vcpu);
        buf = kzalloc(reg_sz, GFP_KERNEL_ACCOUNT);
        if (!buf)
                return -ENOMEM;

        ret = kvm_share_hyp(buf, buf + reg_sz);
        if (ret) {
                kfree(buf);
                return ret;
        }
        
        vcpu->arch.sve_state = buf;
        vcpu_set_flag(vcpu, VCPU_SVE_FINALIZED);
        return 0;
}

int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature)
{
        switch (feature) {
        case KVM_ARM_VCPU_SVE:
                if (!vcpu_has_sve(vcpu))
                        return -EINVAL;

                if (kvm_arm_vcpu_sve_finalized(vcpu))
                        return -EPERM;

                return kvm_vcpu_finalize_sve(vcpu);
        }

        return -EINVAL;
}

bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu)
{
        if (vcpu_has_sve(vcpu) && !kvm_arm_vcpu_sve_finalized(vcpu))
                return false;

        return true;
}

void kvm_arm_vcpu_destroy(struct kvm_vcpu *vcpu)
{
        void *sve_state = vcpu->arch.sve_state;

        kvm_unshare_hyp(vcpu, vcpu + 1);
        if (sve_state)
                kvm_unshare_hyp(sve_state, sve_state + vcpu_sve_state_size(vcpu));
        kfree(sve_state);
        free_page((unsigned long)vcpu->arch.ctxt.vncr_array);
        kfree(vcpu->arch.vncr_tlb);
        kfree(vcpu->arch.ccsidr);
}

static void kvm_vcpu_reset_sve(struct kvm_vcpu *vcpu)
{
        if (vcpu_has_sve(vcpu))
                memset(vcpu->arch.sve_state, 0, vcpu_sve_state_size(vcpu));
}

/**
 * kvm_reset_vcpu - sets core registers and sys_regs to reset value
 * @vcpu: The VCPU pointer
 *
 * This function sets the registers on the virtual CPU struct to their
 * architecturally defined reset values, except for registers whose reset is
 * deferred until kvm_arm_vcpu_finalize().
 *
 * Note: This function can be called from two paths: The KVM_ARM_VCPU_INIT
 * ioctl or as part of handling a request issued by another VCPU in the PSCI
 * handling code.  In the first case, the VCPU will not be loaded, and in the
 * second case the VCPU will be loaded.  Because this function operates purely
 * on the memory-backed values of system registers, we want to do a full put if
 * we were loaded (handling a request) and load the values back at the end of
 * the function.  Otherwise we leave the state alone.  In both cases, we
 * disable preemption around the vcpu reset as we would otherwise race with
 * preempt notifiers which also call put/load.
 */
void kvm_reset_vcpu(struct kvm_vcpu *vcpu)
{
        struct vcpu_reset_state reset_state;
        bool loaded;
        u32 pstate;

        spin_lock(&vcpu->arch.mp_state_lock);
        reset_state = vcpu->arch.reset_state;
        vcpu->arch.reset_state.reset = false;
        spin_unlock(&vcpu->arch.mp_state_lock);

        preempt_disable();
        loaded = (vcpu->cpu != -1);
        if (loaded)
                kvm_arch_vcpu_put(vcpu);

        if (!kvm_arm_vcpu_sve_finalized(vcpu)) {
                if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_SVE))
                        kvm_vcpu_enable_sve(vcpu);
        } else {
                kvm_vcpu_reset_sve(vcpu);
        }

        if (vcpu_el1_is_32bit(vcpu))
                pstate = VCPU_RESET_PSTATE_SVC;
        else if (vcpu_has_nv(vcpu))
                pstate = VCPU_RESET_PSTATE_EL2;
        else
                pstate = VCPU_RESET_PSTATE_EL1;

        /* Reset core registers */
        memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu)));
        memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs));
        vcpu->arch.ctxt.spsr_abt = 0;
        vcpu->arch.ctxt.spsr_und = 0;
        vcpu->arch.ctxt.spsr_irq = 0;
        vcpu->arch.ctxt.spsr_fiq = 0;
        vcpu_gp_regs(vcpu)->pstate = pstate;

        /* Reset system registers */
        kvm_reset_sys_regs(vcpu);

        /*
         * Additional reset state handling that PSCI may have imposed on us.
         * Must be done after all the sys_reg reset.
         */
        if (reset_state.reset) {
                unsigned long target_pc = reset_state.pc;

                /* Gracefully handle Thumb2 entry point */
                if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {
                        target_pc &= ~1UL;
                        vcpu_set_thumb(vcpu);
                }

                /* Propagate caller endianness */
                if (reset_state.be)
                        kvm_vcpu_set_be(vcpu);

                *vcpu_pc(vcpu) = target_pc;
                vcpu_set_reg(vcpu, 0, reset_state.r0);
        }

        /* Reset timer */
        kvm_timer_vcpu_reset(vcpu);

        if (loaded)
                kvm_arch_vcpu_load(vcpu, smp_processor_id());
        preempt_enable();
}

u32 kvm_get_pa_bits(struct kvm *kvm)
{
        /* Fixed limit until we can configure ID_AA64MMFR0.PARange */
        return kvm_ipa_limit;
}

u32 get_kvm_ipa_limit(void)
{
        return kvm_ipa_limit;
}

int __init kvm_set_ipa_limit(void)
{
        unsigned int parange;
        u64 mmfr0;

        mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
        parange = cpuid_feature_extract_unsigned_field(mmfr0,
                                ID_AA64MMFR0_EL1_PARANGE_SHIFT);
        /*
         * IPA size beyond 48 bits for 4K and 16K page size is only supported
         * when LPA2 is available. So if we have LPA2, enable it, else cap to 48
         * bits, in case it's reported as larger on the system.
         */
        if (!kvm_lpa2_is_enabled() && PAGE_SIZE != SZ_64K)
                parange = min(parange, (unsigned int)ID_AA64MMFR0_EL1_PARANGE_48);

        /*
         * Check with ARMv8.5-GTG that our PAGE_SIZE is supported at
         * Stage-2. If not, things will stop very quickly.
         */
        switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_TGRAN_2_SHIFT)) {
        case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE:
                kvm_err("PAGE_SIZE not supported at Stage-2, giving up\n");
                return -EINVAL;
        case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT:
                kvm_debug("PAGE_SIZE supported at Stage-2 (default)\n");
                break;
        case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX:
                kvm_debug("PAGE_SIZE supported at Stage-2 (advertised)\n");
                break;
        default:
                kvm_err("Unsupported value for TGRAN_2, giving up\n");
                return -EINVAL;
        }

        kvm_ipa_limit = id_aa64mmfr0_parange_to_phys_shift(parange);
        kvm_info("IPA Size Limit: %d bits%s\n", kvm_ipa_limit,
                 ((kvm_ipa_limit < KVM_PHYS_SHIFT) ?
                  " (Reduced IPA size, limited VM/VMM compatibility)" : ""));

        return 0;
}












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#ifndef __ASM_SIMD_H
#define __ASM_SIMD_H

#include <linux/compiler.h>
#include <linux/irqflags.h>
#include <linux/percpu.h>
#include <linux/preempt.h>
#include <linux/types.h>

#ifdef CONFIG_KERNEL_MODE_NEON

/*
 * may_use_simd - whether it is allowable at this time to issue SIMD
 *                instructions or access the SIMD register file
 *
 * Callers must not assume that the result remains true beyond the next
 * preempt_enable() or return from softirq context.
 */
static __must_check inline bool may_use_simd(void)
{
        /*
         * We must make sure that the SVE has been initialized properly
         * before using the SIMD in kernel.
         */
        return !WARN_ON(!system_capabilities_finalized()) &&
               system_supports_fpsimd() &&
               !in_hardirq() && !irqs_disabled() && !in_nmi();
}

#else /* ! CONFIG_KERNEL_MODE_NEON */

static __must_check inline bool may_use_simd(void) {
        return false;
}

#endif /* ! CONFIG_KERNEL_MODE_NEON */

#endif
























































































































  287 








  288 









  288 













  288 




  287 





  288 





































































































































































































































































































































































































































































































  198 


  198 













































































    6 

    6 



    6 






































































































































































































































































































































  288 





  288 
  288 




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
// SPDX-License-Identifier: GPL-2.0
/*
 * kobject.c - library routines for handling generic kernel objects
 *
 * Copyright (c) 2002-2003 Patrick Mochel <mochel@osdl.org>
 * Copyright (c) 2006-2007 Greg Kroah-Hartman <greg@kroah.com>
 * Copyright (c) 2006-2007 Novell Inc.
 *
 * Please see the file Documentation/core-api/kobject.rst for critical information
 * about using the kobject interface.
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/kobject.h>
#include <linux/string.h>
#include <linux/export.h>
#include <linux/stat.h>
#include <linux/slab.h>
#include <linux/random.h>

/**
 * kobject_namespace() - Return @kobj's namespace tag.
 * @kobj: kobject in question
 *
 * Returns namespace tag of @kobj if its parent has namespace ops enabled
 * and thus @kobj should have a namespace tag associated with it.  Returns
 * %NULL otherwise.
 */
const void *kobject_namespace(const struct kobject *kobj)
{
        const struct kobj_ns_type_operations *ns_ops = kobj_ns_ops(kobj);

        if (!ns_ops || ns_ops->type == KOBJ_NS_TYPE_NONE)
                return NULL;

        return kobj->ktype->namespace(kobj);
}

/**
 * kobject_get_ownership() - Get sysfs ownership data for @kobj.
 * @kobj: kobject in question
 * @uid: kernel user ID for sysfs objects
 * @gid: kernel group ID for sysfs objects
 *
 * Returns initial uid/gid pair that should be used when creating sysfs
 * representation of given kobject. Normally used to adjust ownership of
 * objects in a container.
 */
void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
        *uid = GLOBAL_ROOT_UID;
        *gid = GLOBAL_ROOT_GID;

        if (kobj->ktype->get_ownership)
                kobj->ktype->get_ownership(kobj, uid, gid);
}

static bool kobj_ns_type_is_valid(enum kobj_ns_type type)
{
        if ((type <= KOBJ_NS_TYPE_NONE) || (type >= KOBJ_NS_TYPES))
                return false;

        return true;
}

static int create_dir(struct kobject *kobj)
{
        const struct kobj_type *ktype = get_ktype(kobj);
        const struct kobj_ns_type_operations *ops;
        int error;

        error = sysfs_create_dir_ns(kobj, kobject_namespace(kobj));
        if (error)
                return error;

        if (ktype) {
                error = sysfs_create_groups(kobj, ktype->default_groups);
                if (error) {
                        sysfs_remove_dir(kobj);
                        return error;
                }
        }

        /*
         * @kobj->sd may be deleted by an ancestor going away.  Hold an
         * extra reference so that it stays until @kobj is gone.
         */
        sysfs_get(kobj->sd);

        /*
         * If @kobj has ns_ops, its children need to be filtered based on
         * their namespace tags.  Enable namespace support on @kobj->sd.
         */
        ops = kobj_child_ns_ops(kobj);
        if (ops) {
                BUG_ON(!kobj_ns_type_is_valid(ops->type));
                BUG_ON(!kobj_ns_type_registered(ops->type));

                sysfs_enable_ns(kobj->sd);
        }

        return 0;
}

static int get_kobj_path_length(const struct kobject *kobj)
{
        int length = 1;
        const struct kobject *parent = kobj;

        /* walk up the ancestors until we hit the one pointing to the
         * root.
         * Add 1 to strlen for leading '/' of each level.
         */
        do {
                if (kobject_name(parent) == NULL)
                        return 0;
                length += strlen(kobject_name(parent)) + 1;
                parent = parent->parent;
        } while (parent);
        return length;
}

static int fill_kobj_path(const struct kobject *kobj, char *path, int length)
{
        const struct kobject *parent;

        --length;
        for (parent = kobj; parent; parent = parent->parent) {
                int cur = strlen(kobject_name(parent));
                /* back up enough to print this name with '/' */
                length -= cur;
                if (length <= 0)
                        return -EINVAL;
                memcpy(path + length, kobject_name(parent), cur);
                *(path + --length) = '/';
        }

        pr_debug("'%s' (%p): %s: path = '%s'\n", kobject_name(kobj),
                 kobj, __func__, path);

        return 0;
}

/**
 * kobject_get_path() - Allocate memory and fill in the path for @kobj.
 * @kobj:        kobject in question, with which to build the path
 * @gfp_mask:        the allocation type used to allocate the path
 *
 * Return: The newly allocated memory, caller must free with kfree().
 */
char *kobject_get_path(const struct kobject *kobj, gfp_t gfp_mask)
{
        char *path;
        int len;

retry:
        len = get_kobj_path_length(kobj);
        if (len == 0)
                return NULL;
        path = kzalloc(len, gfp_mask);
        if (!path)
                return NULL;
        if (fill_kobj_path(kobj, path, len)) {
                kfree(path);
                goto retry;
        }

        return path;
}
EXPORT_SYMBOL_GPL(kobject_get_path);

/* add the kobject to its kset's list */
static void kobj_kset_join(struct kobject *kobj)
{
        if (!kobj->kset)
                return;

        kset_get(kobj->kset);
        spin_lock(&kobj->kset->list_lock);
        list_add_tail(&kobj->entry, &kobj->kset->list);
        spin_unlock(&kobj->kset->list_lock);
}

/* remove the kobject from its kset's list */
static void kobj_kset_leave(struct kobject *kobj)
{
        if (!kobj->kset)
                return;

        spin_lock(&kobj->kset->list_lock);
        list_del_init(&kobj->entry);
        spin_unlock(&kobj->kset->list_lock);
        kset_put(kobj->kset);
}

static void kobject_init_internal(struct kobject *kobj)
{
        if (!kobj)
                return;
        kref_init(&kobj->kref);
        INIT_LIST_HEAD(&kobj->entry);
        kobj->state_in_sysfs = 0;
        kobj->state_add_uevent_sent = 0;
        kobj->state_remove_uevent_sent = 0;
        kobj->state_initialized = 1;
}


static int kobject_add_internal(struct kobject *kobj)
{
        int error = 0;
        struct kobject *parent;

        if (!kobj)
                return -ENOENT;

        if (!kobj->name || !kobj->name[0]) {
                WARN(1,
                     "kobject: (%p): attempted to be registered with empty name!\n",
                     kobj);
                return -EINVAL;
        }

        parent = kobject_get(kobj->parent);

        /* join kset if set, use it as parent if we do not already have one */
        if (kobj->kset) {
                if (!parent)
                        parent = kobject_get(&kobj->kset->kobj);
                kobj_kset_join(kobj);
                kobj->parent = parent;
        }

        pr_debug("'%s' (%p): %s: parent: '%s', set: '%s'\n",
                 kobject_name(kobj), kobj, __func__,
                 parent ? kobject_name(parent) : "<NULL>",
                 kobj->kset ? kobject_name(&kobj->kset->kobj) : "<NULL>");

        error = create_dir(kobj);
        if (error) {
                kobj_kset_leave(kobj);
                kobject_put(parent);
                kobj->parent = NULL;

                /* be noisy on error issues */
                if (error == -EEXIST)
                        pr_err("%s failed for %s with -EEXIST, don't try to register things with the same name in the same directory.\n",
                               __func__, kobject_name(kobj));
                else
                        pr_err("%s failed for %s (error: %d parent: %s)\n",
                               __func__, kobject_name(kobj), error,
                               parent ? kobject_name(parent) : "'none'");
        } else
                kobj->state_in_sysfs = 1;

        return error;
}

/**
 * kobject_set_name_vargs() - Set the name of a kobject.
 * @kobj: struct kobject to set the name of
 * @fmt: format string used to build the name
 * @vargs: vargs to format the string.
 */
int kobject_set_name_vargs(struct kobject *kobj, const char *fmt,
                                  va_list vargs)
{
        const char *s;

        if (kobj->name && !fmt)
                return 0;

        s = kvasprintf_const(GFP_KERNEL, fmt, vargs);
        if (!s)
                return -ENOMEM;

        /*
         * ewww... some of these buggers have '/' in the name ... If
         * that's the case, we need to make sure we have an actual
         * allocated copy to modify, since kvasprintf_const may have
         * returned something from .rodata.
         */
        if (strchr(s, '/')) {
                char *t;

                t = kstrdup(s, GFP_KERNEL);
                kfree_const(s);
                if (!t)
                        return -ENOMEM;
                s = strreplace(t, '/', '!');
        }
        kfree_const(kobj->name);
        kobj->name = s;

        return 0;
}

/**
 * kobject_set_name() - Set the name of a kobject.
 * @kobj: struct kobject to set the name of
 * @fmt: format string used to build the name
 *
 * This sets the name of the kobject.  If you have already added the
 * kobject to the system, you must call kobject_rename() in order to
 * change the name of the kobject.
 */
int kobject_set_name(struct kobject *kobj, const char *fmt, ...)
{
        va_list vargs;
        int retval;

        va_start(vargs, fmt);
        retval = kobject_set_name_vargs(kobj, fmt, vargs);
        va_end(vargs);

        return retval;
}
EXPORT_SYMBOL(kobject_set_name);

/**
 * kobject_init() - Initialize a kobject structure.
 * @kobj: pointer to the kobject to initialize
 * @ktype: pointer to the ktype for this kobject.
 *
 * This function will properly initialize a kobject such that it can then
 * be passed to the kobject_add() call.
 *
 * After this function is called, the kobject MUST be cleaned up by a call
 * to kobject_put(), not by a call to kfree directly to ensure that all of
 * the memory is cleaned up properly.
 */
void kobject_init(struct kobject *kobj, const struct kobj_type *ktype)
{
        char *err_str;

        if (!kobj) {
                err_str = "invalid kobject pointer!";
                goto error;
        }
        if (!ktype) {
                err_str = "must have a ktype to be initialized properly!\n";
                goto error;
        }
        if (kobj->state_initialized) {
                /* do not error out as sometimes we can recover */
                pr_err("kobject (%p): tried to init an initialized object, something is seriously wrong.\n",
                       kobj);
                dump_stack_lvl(KERN_ERR);
        }

        kobject_init_internal(kobj);
        kobj->ktype = ktype;
        return;

error:
        pr_err("kobject (%p): %s\n", kobj, err_str);
        dump_stack_lvl(KERN_ERR);
}
EXPORT_SYMBOL(kobject_init);

static __printf(3, 0) int kobject_add_varg(struct kobject *kobj,
                                           struct kobject *parent,
                                           const char *fmt, va_list vargs)
{
        int retval;

        retval = kobject_set_name_vargs(kobj, fmt, vargs);
        if (retval) {
                pr_err("can not set name properly!\n");
                return retval;
        }
        kobj->parent = parent;
        return kobject_add_internal(kobj);
}

/**
 * kobject_add() - The main kobject add function.
 * @kobj: the kobject to add
 * @parent: pointer to the parent of the kobject.
 * @fmt: format to name the kobject with.
 *
 * The kobject name is set and added to the kobject hierarchy in this
 * function.
 *
 * If @parent is set, then the parent of the @kobj will be set to it.
 * If @parent is NULL, then the parent of the @kobj will be set to the
 * kobject associated with the kset assigned to this kobject.  If no kset
 * is assigned to the kobject, then the kobject will be located in the
 * root of the sysfs tree.
 *
 * Note, no "add" uevent will be created with this call, the caller should set
 * up all of the necessary sysfs files for the object and then call
 * kobject_uevent() with the UEVENT_ADD parameter to ensure that
 * userspace is properly notified of this kobject's creation.
 *
 * Return: If this function returns an error, kobject_put() must be
 *         called to properly clean up the memory associated with the
 *         object.  Under no instance should the kobject that is passed
 *         to this function be directly freed with a call to kfree(),
 *         that can leak memory.
 *
 *         If this function returns success, kobject_put() must also be called
 *         in order to properly clean up the memory associated with the object.
 *
 *         In short, once this function is called, kobject_put() MUST be called
 *         when the use of the object is finished in order to properly free
 *         everything.
 */
int kobject_add(struct kobject *kobj, struct kobject *parent,
                const char *fmt, ...)
{
        va_list args;
        int retval;

        if (!kobj)
                return -EINVAL;

        if (!kobj->state_initialized) {
                pr_err("kobject '%s' (%p): tried to add an uninitialized object, something is seriously wrong.\n",
                       kobject_name(kobj), kobj);
                dump_stack_lvl(KERN_ERR);
                return -EINVAL;
        }
        va_start(args, fmt);
        retval = kobject_add_varg(kobj, parent, fmt, args);
        va_end(args);

        return retval;
}
EXPORT_SYMBOL(kobject_add);

/**
 * kobject_init_and_add() - Initialize a kobject structure and add it to
 *                          the kobject hierarchy.
 * @kobj: pointer to the kobject to initialize
 * @ktype: pointer to the ktype for this kobject.
 * @parent: pointer to the parent of this kobject.
 * @fmt: the name of the kobject.
 *
 * This function combines the call to kobject_init() and kobject_add().
 *
 * If this function returns an error, kobject_put() must be called to
 * properly clean up the memory associated with the object.  This is the
 * same type of error handling after a call to kobject_add() and kobject
 * lifetime rules are the same here.
 */
int kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype,
                         struct kobject *parent, const char *fmt, ...)
{
        va_list args;
        int retval;

        kobject_init(kobj, ktype);

        va_start(args, fmt);
        retval = kobject_add_varg(kobj, parent, fmt, args);
        va_end(args);

        return retval;
}
EXPORT_SYMBOL_GPL(kobject_init_and_add);

/**
 * kobject_rename() - Change the name of an object.
 * @kobj: object in question.
 * @new_name: object's new name
 *
 * It is the responsibility of the caller to provide mutual
 * exclusion between two different calls of kobject_rename
 * on the same kobject and to ensure that new_name is valid and
 * won't conflict with other kobjects.
 */
int kobject_rename(struct kobject *kobj, const char *new_name)
{
        int error = 0;
        const char *devpath = NULL;
        const char *dup_name = NULL, *name;
        char *devpath_string = NULL;
        char *envp[2];

        kobj = kobject_get(kobj);
        if (!kobj)
                return -EINVAL;
        if (!kobj->parent) {
                kobject_put(kobj);
                return -EINVAL;
        }

        devpath = kobject_get_path(kobj, GFP_KERNEL);
        if (!devpath) {
                error = -ENOMEM;
                goto out;
        }
        devpath_string = kmalloc(strlen(devpath) + 15, GFP_KERNEL);
        if (!devpath_string) {
                error = -ENOMEM;
                goto out;
        }
        sprintf(devpath_string, "DEVPATH_OLD=%s", devpath);
        envp[0] = devpath_string;
        envp[1] = NULL;

        name = dup_name = kstrdup_const(new_name, GFP_KERNEL);
        if (!name) {
                error = -ENOMEM;
                goto out;
        }

        error = sysfs_rename_dir_ns(kobj, new_name, kobject_namespace(kobj));
        if (error)
                goto out;

        /* Install the new kobject name */
        dup_name = kobj->name;
        kobj->name = name;

        /* This function is mostly/only used for network interface.
         * Some hotplug package track interfaces by their name and
         * therefore want to know when the name is changed by the user. */
        kobject_uevent_env(kobj, KOBJ_MOVE, envp);

out:
        kfree_const(dup_name);
        kfree(devpath_string);
        kfree(devpath);
        kobject_put(kobj);

        return error;
}
EXPORT_SYMBOL_GPL(kobject_rename);

/**
 * kobject_move() - Move object to another parent.
 * @kobj: object in question.
 * @new_parent: object's new parent (can be NULL)
 */
int kobject_move(struct kobject *kobj, struct kobject *new_parent)
{
        int error;
        struct kobject *old_parent;
        const char *devpath = NULL;
        char *devpath_string = NULL;
        char *envp[2];

        kobj = kobject_get(kobj);
        if (!kobj)
                return -EINVAL;
        new_parent = kobject_get(new_parent);
        if (!new_parent) {
                if (kobj->kset)
                        new_parent = kobject_get(&kobj->kset->kobj);
        }

        /* old object path */
        devpath = kobject_get_path(kobj, GFP_KERNEL);
        if (!devpath) {
                error = -ENOMEM;
                goto out;
        }
        devpath_string = kmalloc(strlen(devpath) + 15, GFP_KERNEL);
        if (!devpath_string) {
                error = -ENOMEM;
                goto out;
        }
        sprintf(devpath_string, "DEVPATH_OLD=%s", devpath);
        envp[0] = devpath_string;
        envp[1] = NULL;
        error = sysfs_move_dir_ns(kobj, new_parent, kobject_namespace(kobj));
        if (error)
                goto out;
        old_parent = kobj->parent;
        kobj->parent = new_parent;
        new_parent = NULL;
        kobject_put(old_parent);
        kobject_uevent_env(kobj, KOBJ_MOVE, envp);
out:
        kobject_put(new_parent);
        kobject_put(kobj);
        kfree(devpath_string);
        kfree(devpath);
        return error;
}
EXPORT_SYMBOL_GPL(kobject_move);

static void __kobject_del(struct kobject *kobj)
{
        struct kernfs_node *sd;
        const struct kobj_type *ktype;

        sd = kobj->sd;
        ktype = get_ktype(kobj);

        if (ktype)
                sysfs_remove_groups(kobj, ktype->default_groups);

        /* send "remove" if the caller did not do it but sent "add" */
        if (kobj->state_add_uevent_sent && !kobj->state_remove_uevent_sent) {
                pr_debug("'%s' (%p): auto cleanup 'remove' event\n",
                         kobject_name(kobj), kobj);
                kobject_uevent(kobj, KOBJ_REMOVE);
        }

        sysfs_remove_dir(kobj);
        sysfs_put(sd);

        kobj->state_in_sysfs = 0;
        kobj_kset_leave(kobj);
        kobj->parent = NULL;
}

/**
 * kobject_del() - Unlink kobject from hierarchy.
 * @kobj: object.
 *
 * This is the function that should be called to delete an object
 * successfully added via kobject_add().
 */
void kobject_del(struct kobject *kobj)
{
        struct kobject *parent;

        if (!kobj)
                return;

        parent = kobj->parent;
        __kobject_del(kobj);
        kobject_put(parent);
}
EXPORT_SYMBOL(kobject_del);

/**
 * kobject_get() - Increment refcount for object.
 * @kobj: object.
 */
struct kobject *kobject_get(struct kobject *kobj)
{
        if (kobj) {
                if (!kobj->state_initialized)
                        WARN(1, KERN_WARNING
                                "kobject: '%s' (%p): is not initialized, yet kobject_get() is being called.\n",
                             kobject_name(kobj), kobj);
                kref_get(&kobj->kref);
        }
        return kobj;
}
EXPORT_SYMBOL(kobject_get);

struct kobject * __must_check kobject_get_unless_zero(struct kobject *kobj)
{
        if (!kobj)
                return NULL;
        if (!kref_get_unless_zero(&kobj->kref))
                kobj = NULL;
        return kobj;
}
EXPORT_SYMBOL(kobject_get_unless_zero);

/*
 * kobject_cleanup - free kobject resources.
 * @kobj: object to cleanup
 */
static void kobject_cleanup(struct kobject *kobj)
{
        struct kobject *parent = kobj->parent;
        const struct kobj_type *t = get_ktype(kobj);
        const char *name = kobj->name;

        pr_debug("'%s' (%p): %s, parent %p\n",
                 kobject_name(kobj), kobj, __func__, kobj->parent);

        if (t && !t->release)
                pr_debug("'%s' (%p): does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.\n",
                         kobject_name(kobj), kobj);

        /* remove from sysfs if the caller did not do it */
        if (kobj->state_in_sysfs) {
                pr_debug("'%s' (%p): auto cleanup kobject_del\n",
                         kobject_name(kobj), kobj);
                __kobject_del(kobj);
        } else {
                /* avoid dropping the parent reference unnecessarily */
                parent = NULL;
        }

        if (t && t->release) {
                pr_debug("'%s' (%p): calling ktype release\n",
                         kobject_name(kobj), kobj);
                t->release(kobj);
        }

        /* free name if we allocated it */
        if (name) {
                pr_debug("'%s': free name\n", name);
                kfree_const(name);
        }

        kobject_put(parent);
}

#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
static void kobject_delayed_cleanup(struct work_struct *work)
{
        kobject_cleanup(container_of(to_delayed_work(work),
                                     struct kobject, release));
}
#endif

static void kobject_release(struct kref *kref)
{
        struct kobject *kobj = container_of(kref, struct kobject, kref);
#ifdef CONFIG_DEBUG_KOBJECT_RELEASE
        unsigned long delay = HZ + HZ * get_random_u32_below(4);
        pr_info("'%s' (%p): %s, parent %p (delayed %ld)\n",
                kobject_name(kobj), kobj, __func__, kobj->parent, delay);
        INIT_DELAYED_WORK(&kobj->release, kobject_delayed_cleanup);

        schedule_delayed_work(&kobj->release, delay);
#else
        kobject_cleanup(kobj);
#endif
}

/**
 * kobject_put() - Decrement refcount for object.
 * @kobj: object.
 *
 * Decrement the refcount, and if 0, call kobject_cleanup().
 */
void kobject_put(struct kobject *kobj)
{
        if (kobj) {
                if (!kobj->state_initialized)
                        WARN(1, KERN_WARNING
                                "kobject: '%s' (%p): is not initialized, yet kobject_put() is being called.\n",
                             kobject_name(kobj), kobj);
                kref_put(&kobj->kref, kobject_release);
        }
}
EXPORT_SYMBOL(kobject_put);

static void dynamic_kobj_release(struct kobject *kobj)
{
        pr_debug("(%p): %s\n", kobj, __func__);
        kfree(kobj);
}

static const struct kobj_type dynamic_kobj_ktype = {
        .release        = dynamic_kobj_release,
        .sysfs_ops        = &kobj_sysfs_ops,
};

/**
 * kobject_create() - Create a struct kobject dynamically.
 *
 * This function creates a kobject structure dynamically and sets it up
 * to be a "dynamic" kobject with a default release function set up.
 *
 * If the kobject was not able to be created, NULL will be returned.
 * The kobject structure returned from here must be cleaned up with a
 * call to kobject_put() and not kfree(), as kobject_init() has
 * already been called on this structure.
 */
static struct kobject *kobject_create(void)
{
        struct kobject *kobj;

        kobj = kzalloc(sizeof(*kobj), GFP_KERNEL);
        if (!kobj)
                return NULL;

        kobject_init(kobj, &dynamic_kobj_ktype);
        return kobj;
}

/**
 * kobject_create_and_add() - Create a struct kobject dynamically and
 *                            register it with sysfs.
 * @name: the name for the kobject
 * @parent: the parent kobject of this kobject, if any.
 *
 * This function creates a kobject structure dynamically and registers it
 * with sysfs.  When you are finished with this structure, call
 * kobject_put() and the structure will be dynamically freed when
 * it is no longer being used.
 *
 * If the kobject was not able to be created, NULL will be returned.
 */
struct kobject *kobject_create_and_add(const char *name, struct kobject *parent)
{
        struct kobject *kobj;
        int retval;

        kobj = kobject_create();
        if (!kobj)
                return NULL;

        retval = kobject_add(kobj, parent, "%s", name);
        if (retval) {
                pr_warn("%s: kobject_add error: %d\n", __func__, retval);
                kobject_put(kobj);
                kobj = NULL;
        }
        return kobj;
}
EXPORT_SYMBOL_GPL(kobject_create_and_add);

/**
 * kset_init() - Initialize a kset for use.
 * @k: kset
 */
void kset_init(struct kset *k)
{
        kobject_init_internal(&k->kobj);
        INIT_LIST_HEAD(&k->list);
        spin_lock_init(&k->list_lock);
}

/* default kobject attribute operations */
static ssize_t kobj_attr_show(struct kobject *kobj, struct attribute *attr,
                              char *buf)
{
        struct kobj_attribute *kattr;
        ssize_t ret = -EIO;

        kattr = container_of(attr, struct kobj_attribute, attr);
        if (kattr->show)
                ret = kattr->show(kobj, kattr, buf);
        return ret;
}

static ssize_t kobj_attr_store(struct kobject *kobj, struct attribute *attr,
                               const char *buf, size_t count)
{
        struct kobj_attribute *kattr;
        ssize_t ret = -EIO;

        kattr = container_of(attr, struct kobj_attribute, attr);
        if (kattr->store)
                ret = kattr->store(kobj, kattr, buf, count);
        return ret;
}

const struct sysfs_ops kobj_sysfs_ops = {
        .show        = kobj_attr_show,
        .store        = kobj_attr_store,
};
EXPORT_SYMBOL_GPL(kobj_sysfs_ops);

/**
 * kset_register() - Initialize and add a kset.
 * @k: kset.
 *
 * NOTE: On error, the kset.kobj.name allocated by() kobj_set_name()
 * is freed, it can not be used any more.
 */
int kset_register(struct kset *k)
{
        int err;

        if (!k)
                return -EINVAL;

        if (!k->kobj.ktype) {
                pr_err("must have a ktype to be initialized properly!\n");
                return -EINVAL;
        }

        kset_init(k);
        err = kobject_add_internal(&k->kobj);
        if (err) {
                kfree_const(k->kobj.name);
                /* Set it to NULL to avoid accessing bad pointer in callers. */
                k->kobj.name = NULL;
                return err;
        }
        kobject_uevent(&k->kobj, KOBJ_ADD);
        return 0;
}
EXPORT_SYMBOL(kset_register);

/**
 * kset_unregister() - Remove a kset.
 * @k: kset.
 */
void kset_unregister(struct kset *k)
{
        if (!k)
                return;
        kobject_del(&k->kobj);
        kobject_put(&k->kobj);
}
EXPORT_SYMBOL(kset_unregister);

/**
 * kset_find_obj() - Search for object in kset.
 * @kset: kset we're looking in.
 * @name: object's name.
 *
 * Lock kset via @kset->subsys, and iterate over @kset->list,
 * looking for a matching kobject. If matching object is found
 * take a reference and return the object.
 */
struct kobject *kset_find_obj(struct kset *kset, const char *name)
{
        struct kobject *k;
        struct kobject *ret = NULL;

        spin_lock(&kset->list_lock);

        list_for_each_entry(k, &kset->list, entry) {
                if (kobject_name(k) && !strcmp(kobject_name(k), name)) {
                        ret = kobject_get_unless_zero(k);
                        break;
                }
        }

        spin_unlock(&kset->list_lock);
        return ret;
}
EXPORT_SYMBOL_GPL(kset_find_obj);

static void kset_release(struct kobject *kobj)
{
        struct kset *kset = container_of(kobj, struct kset, kobj);
        pr_debug("'%s' (%p): %s\n",
                 kobject_name(kobj), kobj, __func__);
        kfree(kset);
}

static void kset_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid)
{
        if (kobj->parent)
                kobject_get_ownership(kobj->parent, uid, gid);
}

static const struct kobj_type kset_ktype = {
        .sysfs_ops        = &kobj_sysfs_ops,
        .release        = kset_release,
        .get_ownership        = kset_get_ownership,
};

/**
 * kset_create() - Create a struct kset dynamically.
 *
 * @name: the name for the kset
 * @uevent_ops: a struct kset_uevent_ops for the kset
 * @parent_kobj: the parent kobject of this kset, if any.
 *
 * This function creates a kset structure dynamically.  This structure can
 * then be registered with the system and show up in sysfs with a call to
 * kset_register().  When you are finished with this structure, if
 * kset_register() has been called, call kset_unregister() and the
 * structure will be dynamically freed when it is no longer being used.
 *
 * If the kset was not able to be created, NULL will be returned.
 */
static struct kset *kset_create(const char *name,
                                const struct kset_uevent_ops *uevent_ops,
                                struct kobject *parent_kobj)
{
        struct kset *kset;
        int retval;

        kset = kzalloc(sizeof(*kset), GFP_KERNEL);
        if (!kset)
                return NULL;
        retval = kobject_set_name(&kset->kobj, "%s", name);
        if (retval) {
                kfree(kset);
                return NULL;
        }
        kset->uevent_ops = uevent_ops;
        kset->kobj.parent = parent_kobj;

        /*
         * The kobject of this kset will have a type of kset_ktype and belong to
         * no kset itself.  That way we can properly free it when it is
         * finished being used.
         */
        kset->kobj.ktype = &kset_ktype;
        kset->kobj.kset = NULL;

        return kset;
}

/**
 * kset_create_and_add() - Create a struct kset dynamically and add it to sysfs.
 *
 * @name: the name for the kset
 * @uevent_ops: a struct kset_uevent_ops for the kset
 * @parent_kobj: the parent kobject of this kset, if any.
 *
 * This function creates a kset structure dynamically and registers it
 * with sysfs.  When you are finished with this structure, call
 * kset_unregister() and the structure will be dynamically freed when it
 * is no longer being used.
 *
 * If the kset was not able to be created, NULL will be returned.
 */
struct kset *kset_create_and_add(const char *name,
                                 const struct kset_uevent_ops *uevent_ops,
                                 struct kobject *parent_kobj)
{
        struct kset *kset;
        int error;

        kset = kset_create(name, uevent_ops, parent_kobj);
        if (!kset)
                return NULL;
        error = kset_register(kset);
        if (error) {
                kfree(kset);
                return NULL;
        }
        return kset;
}
EXPORT_SYMBOL_GPL(kset_create_and_add);


static DEFINE_SPINLOCK(kobj_ns_type_lock);
static const struct kobj_ns_type_operations *kobj_ns_ops_tbl[KOBJ_NS_TYPES];

int kobj_ns_type_register(const struct kobj_ns_type_operations *ops)
{
        enum kobj_ns_type type = ops->type;
        int error;

        spin_lock(&kobj_ns_type_lock);

        error = -EINVAL;
        if (!kobj_ns_type_is_valid(type))
                goto out;

        error = -EBUSY;
        if (kobj_ns_ops_tbl[type])
                goto out;

        error = 0;
        kobj_ns_ops_tbl[type] = ops;

out:
        spin_unlock(&kobj_ns_type_lock);
        return error;
}

int kobj_ns_type_registered(enum kobj_ns_type type)
{
        int registered = 0;

        spin_lock(&kobj_ns_type_lock);
        if (kobj_ns_type_is_valid(type))
                registered = kobj_ns_ops_tbl[type] != NULL;
        spin_unlock(&kobj_ns_type_lock);

        return registered;
}

const struct kobj_ns_type_operations *kobj_child_ns_ops(const struct kobject *parent)
{
        const struct kobj_ns_type_operations *ops = NULL;

        if (parent && parent->ktype && parent->ktype->child_ns_type)
                ops = parent->ktype->child_ns_type(parent);

        return ops;
}

const struct kobj_ns_type_operations *kobj_ns_ops(const struct kobject *kobj)
{
        return kobj_child_ns_ops(kobj->parent);
}

bool kobj_ns_current_may_mount(enum kobj_ns_type type)
{
        bool may_mount = true;

        spin_lock(&kobj_ns_type_lock);
        if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
                may_mount = kobj_ns_ops_tbl[type]->current_may_mount();
        spin_unlock(&kobj_ns_type_lock);

        return may_mount;
}

void *kobj_ns_grab_current(enum kobj_ns_type type)
{
        void *ns = NULL;

        spin_lock(&kobj_ns_type_lock);
        if (kobj_ns_type_is_valid(type) && kobj_ns_ops_tbl[type])
                ns = kobj_ns_ops_tbl[type]->grab_current_ns();
        spin_unlock(&kobj_ns_type_lock);

        return ns;
}
EXPORT_SYMBOL_GPL(kobj_ns_grab_current);

void kobj_ns_drop(enum kobj_ns_type type, void *ns)
{
        spin_lock(&kobj_ns_type_lock);
        if (kobj_ns_type_is_valid(type) &&
            kobj_ns_ops_tbl[type] && kobj_ns_ops_tbl[type]->drop_ns)
                kobj_ns_ops_tbl[type]->drop_ns(ns);
        spin_unlock(&kobj_ns_type_lock);
}
EXPORT_SYMBOL_GPL(kobj_ns_drop);























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
 * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
 * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
 * Copyright (c) 2004, 2020 Intel Corporation.  All rights reserved.
 * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
 * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
 * Copyright (c) 2005, 2006, 2007 Cisco Systems.  All rights reserved.
 */

#ifndef IB_VERBS_H
#define IB_VERBS_H

#include <linux/ethtool.h>
#include <linux/types.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/rwsem.h>
#include <linux/workqueue.h>
#include <linux/irq_poll.h>
#include <uapi/linux/if_ether.h>
#include <net/ipv6.h>
#include <net/ip.h>
#include <linux/string.h>
#include <linux/slab.h>
#include <linux/netdevice.h>
#include <linux/refcount.h>
#include <linux/if_link.h>
#include <linux/atomic.h>
#include <linux/mmu_notifier.h>
#include <linux/uaccess.h>
#include <linux/cgroup_rdma.h>
#include <linux/irqflags.h>
#include <linux/preempt.h>
#include <linux/dim.h>
#include <uapi/rdma/ib_user_verbs.h>
#include <rdma/rdma_counter.h>
#include <rdma/restrack.h>
#include <rdma/signature.h>
#include <uapi/rdma/rdma_user_ioctl.h>
#include <uapi/rdma/ib_user_ioctl_verbs.h>

#define IB_FW_VERSION_NAME_MAX        ETHTOOL_FWVERS_LEN

struct ib_umem_odp;
struct ib_uqp_object;
struct ib_usrq_object;
struct ib_uwq_object;
struct rdma_cm_id;
struct ib_port;
struct hw_stats_device_data;

extern struct workqueue_struct *ib_wq;
extern struct workqueue_struct *ib_comp_wq;
extern struct workqueue_struct *ib_comp_unbound_wq;

struct ib_ucq_object;

__printf(2, 3) __cold
void ibdev_emerg(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_alert(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_crit(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_err(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_warn(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_notice(const struct ib_device *ibdev, const char *format, ...);
__printf(2, 3) __cold
void ibdev_info(const struct ib_device *ibdev, const char *format, ...);

#if defined(CONFIG_DYNAMIC_DEBUG) || \
        (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
#define ibdev_dbg(__dev, format, args...)                       \
        dynamic_ibdev_dbg(__dev, format, ##args)
#else
__printf(2, 3) __cold
static inline
void ibdev_dbg(const struct ib_device *ibdev, const char *format, ...) {}
#endif

#define ibdev_level_ratelimited(ibdev_level, ibdev, fmt, ...)           \
do {                                                                    \
        static DEFINE_RATELIMIT_STATE(_rs,                              \
                                      DEFAULT_RATELIMIT_INTERVAL,       \
                                      DEFAULT_RATELIMIT_BURST);         \
        if (__ratelimit(&_rs))                                          \
                ibdev_level(ibdev, fmt, ##__VA_ARGS__);                 \
} while (0)

#define ibdev_emerg_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_emerg, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_alert_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_alert, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_crit_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_crit, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_err_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_err, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_warn_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_warn, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_notice_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_notice, ibdev, fmt, ##__VA_ARGS__)
#define ibdev_info_ratelimited(ibdev, fmt, ...) \
        ibdev_level_ratelimited(ibdev_info, ibdev, fmt, ##__VA_ARGS__)

#if defined(CONFIG_DYNAMIC_DEBUG) || \
        (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
/* descriptor check is first to prevent flooding with "callbacks suppressed" */
#define ibdev_dbg_ratelimited(ibdev, fmt, ...)                          \
do {                                                                    \
        static DEFINE_RATELIMIT_STATE(_rs,                              \
                                      DEFAULT_RATELIMIT_INTERVAL,       \
                                      DEFAULT_RATELIMIT_BURST);         \
        DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, fmt);                 \
        if (DYNAMIC_DEBUG_BRANCH(descriptor) && __ratelimit(&_rs))      \
                __dynamic_ibdev_dbg(&descriptor, ibdev, fmt,            \
                                    ##__VA_ARGS__);                     \
} while (0)
#else
__printf(2, 3) __cold
static inline
void ibdev_dbg_ratelimited(const struct ib_device *ibdev, const char *format, ...) {}
#endif

union ib_gid {
        u8        raw[16];
        struct {
                __be64        subnet_prefix;
                __be64        interface_id;
        } global;
};

extern union ib_gid zgid;

enum ib_gid_type {
        IB_GID_TYPE_IB = IB_UVERBS_GID_TYPE_IB,
        IB_GID_TYPE_ROCE = IB_UVERBS_GID_TYPE_ROCE_V1,
        IB_GID_TYPE_ROCE_UDP_ENCAP = IB_UVERBS_GID_TYPE_ROCE_V2,
        IB_GID_TYPE_SIZE
};

#define ROCE_V2_UDP_DPORT      4791
struct ib_gid_attr {
        struct net_device __rcu        *ndev;
        struct ib_device        *device;
        union ib_gid                gid;
        enum ib_gid_type        gid_type;
        u16                        index;
        u32                        port_num;
};

enum {
        /* set the local administered indication */
        IB_SA_WELL_KNOWN_GUID        = BIT_ULL(57) | 2,
};

enum rdma_transport_type {
        RDMA_TRANSPORT_IB,
        RDMA_TRANSPORT_IWARP,
        RDMA_TRANSPORT_USNIC,
        RDMA_TRANSPORT_USNIC_UDP,
        RDMA_TRANSPORT_UNSPECIFIED,
};

enum rdma_protocol_type {
        RDMA_PROTOCOL_IB,
        RDMA_PROTOCOL_IBOE,
        RDMA_PROTOCOL_IWARP,
        RDMA_PROTOCOL_USNIC_UDP
};

__attribute_const__ enum rdma_transport_type
rdma_node_get_transport(unsigned int node_type);

enum rdma_network_type {
        RDMA_NETWORK_IB,
        RDMA_NETWORK_ROCE_V1,
        RDMA_NETWORK_IPV4,
        RDMA_NETWORK_IPV6
};

static inline enum ib_gid_type ib_network_to_gid_type(enum rdma_network_type network_type)
{
        if (network_type == RDMA_NETWORK_IPV4 ||
            network_type == RDMA_NETWORK_IPV6)
                return IB_GID_TYPE_ROCE_UDP_ENCAP;
        else if (network_type == RDMA_NETWORK_ROCE_V1)
                return IB_GID_TYPE_ROCE;
        else
                return IB_GID_TYPE_IB;
}

static inline enum rdma_network_type
rdma_gid_attr_network_type(const struct ib_gid_attr *attr)
{
        if (attr->gid_type == IB_GID_TYPE_IB)
                return RDMA_NETWORK_IB;

        if (attr->gid_type == IB_GID_TYPE_ROCE)
                return RDMA_NETWORK_ROCE_V1;

        if (ipv6_addr_v4mapped((struct in6_addr *)&attr->gid))
                return RDMA_NETWORK_IPV4;
        else
                return RDMA_NETWORK_IPV6;
}

enum rdma_link_layer {
        IB_LINK_LAYER_UNSPECIFIED,
        IB_LINK_LAYER_INFINIBAND,
        IB_LINK_LAYER_ETHERNET,
};

enum ib_device_cap_flags {
        IB_DEVICE_RESIZE_MAX_WR = IB_UVERBS_DEVICE_RESIZE_MAX_WR,
        IB_DEVICE_BAD_PKEY_CNTR = IB_UVERBS_DEVICE_BAD_PKEY_CNTR,
        IB_DEVICE_BAD_QKEY_CNTR = IB_UVERBS_DEVICE_BAD_QKEY_CNTR,
        IB_DEVICE_RAW_MULTI = IB_UVERBS_DEVICE_RAW_MULTI,
        IB_DEVICE_AUTO_PATH_MIG = IB_UVERBS_DEVICE_AUTO_PATH_MIG,
        IB_DEVICE_CHANGE_PHY_PORT = IB_UVERBS_DEVICE_CHANGE_PHY_PORT,
        IB_DEVICE_UD_AV_PORT_ENFORCE = IB_UVERBS_DEVICE_UD_AV_PORT_ENFORCE,
        IB_DEVICE_CURR_QP_STATE_MOD = IB_UVERBS_DEVICE_CURR_QP_STATE_MOD,
        IB_DEVICE_SHUTDOWN_PORT = IB_UVERBS_DEVICE_SHUTDOWN_PORT,
        /* IB_DEVICE_INIT_TYPE = IB_UVERBS_DEVICE_INIT_TYPE, (not in use) */
        IB_DEVICE_PORT_ACTIVE_EVENT = IB_UVERBS_DEVICE_PORT_ACTIVE_EVENT,
        IB_DEVICE_SYS_IMAGE_GUID = IB_UVERBS_DEVICE_SYS_IMAGE_GUID,
        IB_DEVICE_RC_RNR_NAK_GEN = IB_UVERBS_DEVICE_RC_RNR_NAK_GEN,
        IB_DEVICE_SRQ_RESIZE = IB_UVERBS_DEVICE_SRQ_RESIZE,
        IB_DEVICE_N_NOTIFY_CQ = IB_UVERBS_DEVICE_N_NOTIFY_CQ,

        /* Reserved, old SEND_W_INV = 1 << 16,*/
        IB_DEVICE_MEM_WINDOW = IB_UVERBS_DEVICE_MEM_WINDOW,
        /*
         * Devices should set IB_DEVICE_UD_IP_SUM if they support
         * insertion of UDP and TCP checksum on outgoing UD IPoIB
         * messages and can verify the validity of checksum for
         * incoming messages.  Setting this flag implies that the
         * IPoIB driver may set NETIF_F_IP_CSUM for datagram mode.
         */
        IB_DEVICE_UD_IP_CSUM = IB_UVERBS_DEVICE_UD_IP_CSUM,
        IB_DEVICE_XRC = IB_UVERBS_DEVICE_XRC,

        /*
         * This device supports the IB "base memory management extension",
         * which includes support for fast registrations (IB_WR_REG_MR,
         * IB_WR_LOCAL_INV and IB_WR_SEND_WITH_INV verbs).  This flag should
         * also be set by any iWarp device which must support FRs to comply
         * to the iWarp verbs spec.  iWarp devices also support the
         * IB_WR_RDMA_READ_WITH_INV verb for RDMA READs that invalidate the
         * stag.
         */
        IB_DEVICE_MEM_MGT_EXTENSIONS = IB_UVERBS_DEVICE_MEM_MGT_EXTENSIONS,
        IB_DEVICE_MEM_WINDOW_TYPE_2A = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2A,
        IB_DEVICE_MEM_WINDOW_TYPE_2B = IB_UVERBS_DEVICE_MEM_WINDOW_TYPE_2B,
        IB_DEVICE_RC_IP_CSUM = IB_UVERBS_DEVICE_RC_IP_CSUM,
        /* Deprecated. Please use IB_RAW_PACKET_CAP_IP_CSUM. */
        IB_DEVICE_RAW_IP_CSUM = IB_UVERBS_DEVICE_RAW_IP_CSUM,
        IB_DEVICE_MANAGED_FLOW_STEERING =
                IB_UVERBS_DEVICE_MANAGED_FLOW_STEERING,
        /* Deprecated. Please use IB_RAW_PACKET_CAP_SCATTER_FCS. */
        IB_DEVICE_RAW_SCATTER_FCS = IB_UVERBS_DEVICE_RAW_SCATTER_FCS,
        /* The device supports padding incoming writes to cacheline. */
        IB_DEVICE_PCI_WRITE_END_PADDING =
                IB_UVERBS_DEVICE_PCI_WRITE_END_PADDING,
        /* Placement type attributes */
        IB_DEVICE_FLUSH_GLOBAL = IB_UVERBS_DEVICE_FLUSH_GLOBAL,
        IB_DEVICE_FLUSH_PERSISTENT = IB_UVERBS_DEVICE_FLUSH_PERSISTENT,
        IB_DEVICE_ATOMIC_WRITE = IB_UVERBS_DEVICE_ATOMIC_WRITE,
};

enum ib_kernel_cap_flags {
        /*
         * This device supports a per-device lkey or stag that can be
         * used without performing a memory registration for the local
         * memory.  Note that ULPs should never check this flag, but
         * instead of use the local_dma_lkey flag in the ib_pd structure,
         * which will always contain a usable lkey.
         */
        IBK_LOCAL_DMA_LKEY = 1 << 0,
        /* IB_QP_CREATE_INTEGRITY_EN is supported to implement T10-PI */
        IBK_INTEGRITY_HANDOVER = 1 << 1,
        /* IB_ACCESS_ON_DEMAND is supported during reg_user_mr() */
        IBK_ON_DEMAND_PAGING = 1 << 2,
        /* IB_MR_TYPE_SG_GAPS is supported */
        IBK_SG_GAPS_REG = 1 << 3,
        /* Driver supports RDMA_NLDEV_CMD_DELLINK */
        IBK_ALLOW_USER_UNREG = 1 << 4,

        /* ipoib will use IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK */
        IBK_BLOCK_MULTICAST_LOOPBACK = 1 << 5,
        /* iopib will use IB_QP_CREATE_IPOIB_UD_LSO for its QPs */
        IBK_UD_TSO = 1 << 6,
        /* iopib will use the device ops:
         *   get_vf_config
         *   get_vf_guid
         *   get_vf_stats
         *   set_vf_guid
         *   set_vf_link_state
         */
        IBK_VIRTUAL_FUNCTION = 1 << 7,
        /* ipoib will use IB_QP_CREATE_NETDEV_USE for its QPs */
        IBK_RDMA_NETDEV_OPA = 1 << 8,
};

enum ib_atomic_cap {
        IB_ATOMIC_NONE,
        IB_ATOMIC_HCA,
        IB_ATOMIC_GLOB
};

enum ib_odp_general_cap_bits {
        IB_ODP_SUPPORT                = IB_UVERBS_ODP_SUPPORT,
        IB_ODP_SUPPORT_IMPLICIT = IB_UVERBS_ODP_SUPPORT_IMPLICIT,
};

enum ib_odp_transport_cap_bits {
        IB_ODP_SUPPORT_SEND        = IB_UVERBS_ODP_SUPPORT_SEND,
        IB_ODP_SUPPORT_RECV        = IB_UVERBS_ODP_SUPPORT_RECV,
        IB_ODP_SUPPORT_WRITE        = IB_UVERBS_ODP_SUPPORT_WRITE,
        IB_ODP_SUPPORT_READ        = IB_UVERBS_ODP_SUPPORT_READ,
        IB_ODP_SUPPORT_ATOMIC        = IB_UVERBS_ODP_SUPPORT_ATOMIC,
        IB_ODP_SUPPORT_SRQ_RECV        = IB_UVERBS_ODP_SUPPORT_SRQ_RECV,
        IB_ODP_SUPPORT_FLUSH        = IB_UVERBS_ODP_SUPPORT_FLUSH,
        IB_ODP_SUPPORT_ATOMIC_WRITE        = IB_UVERBS_ODP_SUPPORT_ATOMIC_WRITE,
};

struct ib_odp_caps {
        uint64_t general_caps;
        struct {
                uint32_t  rc_odp_caps;
                uint32_t  uc_odp_caps;
                uint32_t  ud_odp_caps;
                uint32_t  xrc_odp_caps;
        } per_transport_caps;
};

struct ib_rss_caps {
        /* Corresponding bit will be set if qp type from
         * 'enum ib_qp_type' is supported, e.g.
         * supported_qpts |= 1 << IB_QPT_UD
         */
        u32 supported_qpts;
        u32 max_rwq_indirection_tables;
        u32 max_rwq_indirection_table_size;
};

enum ib_tm_cap_flags {
        /*  Support tag matching with rendezvous offload for RC transport */
        IB_TM_CAP_RNDV_RC = 1 << 0,
};

struct ib_tm_caps {
        /* Max size of RNDV header */
        u32 max_rndv_hdr_size;
        /* Max number of entries in tag matching list */
        u32 max_num_tags;
        /* From enum ib_tm_cap_flags */
        u32 flags;
        /* Max number of outstanding list operations */
        u32 max_ops;
        /* Max number of SGE in tag matching entry */
        u32 max_sge;
};

struct ib_cq_init_attr {
        unsigned int        cqe;
        u32                comp_vector;
        u32                flags;
};

enum ib_cq_attr_mask {
        IB_CQ_MODERATE = 1 << 0,
};

struct ib_cq_caps {
        u16     max_cq_moderation_count;
        u16     max_cq_moderation_period;
};

struct ib_dm_mr_attr {
        u64                length;
        u64                offset;
        u32                access_flags;
};

struct ib_dm_alloc_attr {
        u64        length;
        u32        alignment;
        u32        flags;
};

struct ib_device_attr {
        u64                        fw_ver;
        __be64                        sys_image_guid;
        u64                        max_mr_size;
        u64                        page_size_cap;
        u32                        vendor_id;
        u32                        vendor_part_id;
        u32                        hw_ver;
        int                        max_qp;
        int                        max_qp_wr;
        u64                        device_cap_flags;
        u64                        kernel_cap_flags;
        int                        max_send_sge;
        int                        max_recv_sge;
        int                        max_sge_rd;
        int                        max_cq;
        int                        max_cqe;
        int                        max_mr;
        int                        max_pd;
        int                        max_qp_rd_atom;
        int                        max_ee_rd_atom;
        int                        max_res_rd_atom;
        int                        max_qp_init_rd_atom;
        int                        max_ee_init_rd_atom;
        enum ib_atomic_cap        atomic_cap;
        enum ib_atomic_cap        masked_atomic_cap;
        int                        max_ee;
        int                        max_rdd;
        int                        max_mw;
        int                        max_raw_ipv6_qp;
        int                        max_raw_ethy_qp;
        int                        max_mcast_grp;
        int                        max_mcast_qp_attach;
        int                        max_total_mcast_qp_attach;
        int                        max_ah;
        int                        max_srq;
        int                        max_srq_wr;
        int                        max_srq_sge;
        unsigned int                max_fast_reg_page_list_len;
        unsigned int                max_pi_fast_reg_page_list_len;
        u16                        max_pkeys;
        u8                        local_ca_ack_delay;
        int                        sig_prot_cap;
        int                        sig_guard_cap;
        struct ib_odp_caps        odp_caps;
        uint64_t                timestamp_mask;
        uint64_t                hca_core_clock; /* in KHZ */
        struct ib_rss_caps        rss_caps;
        u32                        max_wq_type_rq;
        u32                        raw_packet_caps; /* Use ib_raw_packet_caps enum */
        struct ib_tm_caps        tm_caps;
        struct ib_cq_caps       cq_caps;
        u64                        max_dm_size;
        /* Max entries for sgl for optimized performance per READ */
        u32                        max_sgl_rd;
};

enum ib_mtu {
        IB_MTU_256  = 1,
        IB_MTU_512  = 2,
        IB_MTU_1024 = 3,
        IB_MTU_2048 = 4,
        IB_MTU_4096 = 5
};

enum opa_mtu {
        OPA_MTU_8192 = 6,
        OPA_MTU_10240 = 7
};

static inline int ib_mtu_enum_to_int(enum ib_mtu mtu)
{
        switch (mtu) {
        case IB_MTU_256:  return  256;
        case IB_MTU_512:  return  512;
        case IB_MTU_1024: return 1024;
        case IB_MTU_2048: return 2048;
        case IB_MTU_4096: return 4096;
        default:           return -1;
        }
}

static inline enum ib_mtu ib_mtu_int_to_enum(int mtu)
{
        if (mtu >= 4096)
                return IB_MTU_4096;
        else if (mtu >= 2048)
                return IB_MTU_2048;
        else if (mtu >= 1024)
                return IB_MTU_1024;
        else if (mtu >= 512)
                return IB_MTU_512;
        else
                return IB_MTU_256;
}

static inline int opa_mtu_enum_to_int(enum opa_mtu mtu)
{
        switch (mtu) {
        case OPA_MTU_8192:
                return 8192;
        case OPA_MTU_10240:
                return 10240;
        default:
                return(ib_mtu_enum_to_int((enum ib_mtu)mtu));
        }
}

static inline enum opa_mtu opa_mtu_int_to_enum(int mtu)
{
        if (mtu >= 10240)
                return OPA_MTU_10240;
        else if (mtu >= 8192)
                return OPA_MTU_8192;
        else
                return ((enum opa_mtu)ib_mtu_int_to_enum(mtu));
}

enum ib_port_state {
        IB_PORT_NOP                = 0,
        IB_PORT_DOWN                = 1,
        IB_PORT_INIT                = 2,
        IB_PORT_ARMED                = 3,
        IB_PORT_ACTIVE                = 4,
        IB_PORT_ACTIVE_DEFER        = 5
};

static inline const char *__attribute_const__
ib_port_state_to_str(enum ib_port_state state)
{
        const char * const states[] = {
                [IB_PORT_NOP] = "NOP",
                [IB_PORT_DOWN] = "DOWN",
                [IB_PORT_INIT] = "INIT",
                [IB_PORT_ARMED] = "ARMED",
                [IB_PORT_ACTIVE] = "ACTIVE",
                [IB_PORT_ACTIVE_DEFER] = "ACTIVE_DEFER",
        };

        if (state < ARRAY_SIZE(states))
                return states[state];
        return "UNKNOWN";
}

enum ib_port_phys_state {
        IB_PORT_PHYS_STATE_SLEEP = 1,
        IB_PORT_PHYS_STATE_POLLING = 2,
        IB_PORT_PHYS_STATE_DISABLED = 3,
        IB_PORT_PHYS_STATE_PORT_CONFIGURATION_TRAINING = 4,
        IB_PORT_PHYS_STATE_LINK_UP = 5,
        IB_PORT_PHYS_STATE_LINK_ERROR_RECOVERY = 6,
        IB_PORT_PHYS_STATE_PHY_TEST = 7,
};

enum ib_port_width {
        IB_WIDTH_1X        = 1,
        IB_WIDTH_2X        = 16,
        IB_WIDTH_4X        = 2,
        IB_WIDTH_8X        = 4,
        IB_WIDTH_12X        = 8
};

static inline int ib_width_enum_to_int(enum ib_port_width width)
{
        switch (width) {
        case IB_WIDTH_1X:  return  1;
        case IB_WIDTH_2X:  return  2;
        case IB_WIDTH_4X:  return  4;
        case IB_WIDTH_8X:  return  8;
        case IB_WIDTH_12X: return 12;
        default:           return -1;
        }
}

enum ib_port_speed {
        IB_SPEED_SDR        = 1,
        IB_SPEED_DDR        = 2,
        IB_SPEED_QDR        = 4,
        IB_SPEED_FDR10        = 8,
        IB_SPEED_FDR        = 16,
        IB_SPEED_EDR        = 32,
        IB_SPEED_HDR        = 64,
        IB_SPEED_NDR        = 128,
        IB_SPEED_XDR        = 256,
};

enum ib_stat_flag {
        IB_STAT_FLAG_OPTIONAL = 1 << 0,
};

/**
 * struct rdma_stat_desc
 * @name - The name of the counter
 * @flags - Flags of the counter; For example, IB_STAT_FLAG_OPTIONAL
 * @priv - Driver private information; Core code should not use
 */
struct rdma_stat_desc {
        const char *name;
        unsigned int flags;
        const void *priv;
};

/**
 * struct rdma_hw_stats
 * @lock - Mutex to protect parallel write access to lifespan and values
 *    of counters, which are 64bits and not guaranteed to be written
 *    atomicaly on 32bits systems.
 * @timestamp - Used by the core code to track when the last update was
 * @lifespan - Used by the core code to determine how old the counters
 *   should be before being updated again.  Stored in jiffies, defaults
 *   to 10 milliseconds, drivers can override the default be specifying
 *   their own value during their allocation routine.
 * @descs - Array of pointers to static descriptors used for the counters
 *   in directory.
 * @is_disabled - A bitmap to indicate each counter is currently disabled
 *   or not.
 * @num_counters - How many hardware counters there are.  If name is
 *   shorter than this number, a kernel oops will result.  Driver authors
 *   are encouraged to leave BUILD_BUG_ON(ARRAY_SIZE(@name) < num_counters)
 *   in their code to prevent this.
 * @value - Array of u64 counters that are accessed by the sysfs code and
 *   filled in by the drivers get_stats routine
 */
struct rdma_hw_stats {
        struct mutex        lock; /* Protect lifespan and values[] */
        unsigned long        timestamp;
        unsigned long        lifespan;
        const struct rdma_stat_desc *descs;
        unsigned long        *is_disabled;
        int                num_counters;
        u64                value[] __counted_by(num_counters);
};

#define RDMA_HW_STATS_DEFAULT_LIFESPAN 10

struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
        const struct rdma_stat_desc *descs, int num_counters,
        unsigned long lifespan);

void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats);

/* Define bits for the various functionality this port needs to be supported by
 * the core.
 */
/* Management                           0x00000FFF */
#define RDMA_CORE_CAP_IB_MAD            0x00000001
#define RDMA_CORE_CAP_IB_SMI            0x00000002
#define RDMA_CORE_CAP_IB_CM             0x00000004
#define RDMA_CORE_CAP_IW_CM             0x00000008
#define RDMA_CORE_CAP_IB_SA             0x00000010
#define RDMA_CORE_CAP_OPA_MAD           0x00000020

/* Address format                       0x000FF000 */
#define RDMA_CORE_CAP_AF_IB             0x00001000
#define RDMA_CORE_CAP_ETH_AH            0x00002000
#define RDMA_CORE_CAP_OPA_AH            0x00004000
#define RDMA_CORE_CAP_IB_GRH_REQUIRED   0x00008000

/* Protocol                             0xFFF00000 */
#define RDMA_CORE_CAP_PROT_IB           0x00100000
#define RDMA_CORE_CAP_PROT_ROCE         0x00200000
#define RDMA_CORE_CAP_PROT_IWARP        0x00400000
#define RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP 0x00800000
#define RDMA_CORE_CAP_PROT_RAW_PACKET   0x01000000
#define RDMA_CORE_CAP_PROT_USNIC        0x02000000

#define RDMA_CORE_PORT_IB_GRH_REQUIRED (RDMA_CORE_CAP_IB_GRH_REQUIRED \
                                        | RDMA_CORE_CAP_PROT_ROCE     \
                                        | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP)

#define RDMA_CORE_PORT_IBA_IB          (RDMA_CORE_CAP_PROT_IB  \
                                        | RDMA_CORE_CAP_IB_MAD \
                                        | RDMA_CORE_CAP_IB_SMI \
                                        | RDMA_CORE_CAP_IB_CM  \
                                        | RDMA_CORE_CAP_IB_SA  \
                                        | RDMA_CORE_CAP_AF_IB)
#define RDMA_CORE_PORT_IBA_ROCE        (RDMA_CORE_CAP_PROT_ROCE \
                                        | RDMA_CORE_CAP_IB_MAD  \
                                        | RDMA_CORE_CAP_IB_CM   \
                                        | RDMA_CORE_CAP_AF_IB   \
                                        | RDMA_CORE_CAP_ETH_AH)
#define RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP                        \
                                        (RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP \
                                        | RDMA_CORE_CAP_IB_MAD  \
                                        | RDMA_CORE_CAP_IB_CM   \
                                        | RDMA_CORE_CAP_AF_IB   \
                                        | RDMA_CORE_CAP_ETH_AH)
#define RDMA_CORE_PORT_IWARP           (RDMA_CORE_CAP_PROT_IWARP \
                                        | RDMA_CORE_CAP_IW_CM)
#define RDMA_CORE_PORT_INTEL_OPA       (RDMA_CORE_PORT_IBA_IB  \
                                        | RDMA_CORE_CAP_OPA_MAD)

#define RDMA_CORE_PORT_RAW_PACKET        (RDMA_CORE_CAP_PROT_RAW_PACKET)

#define RDMA_CORE_PORT_USNIC                (RDMA_CORE_CAP_PROT_USNIC)

struct ib_port_attr {
        u64                        subnet_prefix;
        enum ib_port_state        state;
        enum ib_mtu                max_mtu;
        enum ib_mtu                active_mtu;
        u32                     phys_mtu;
        int                        gid_tbl_len;
        unsigned int                ip_gids:1;
        /* This is the value from PortInfo CapabilityMask, defined by IBA */
        u32                        port_cap_flags;
        u32                        max_msg_sz;
        u32                        bad_pkey_cntr;
        u32                        qkey_viol_cntr;
        u16                        pkey_tbl_len;
        u32                        sm_lid;
        u32                        lid;
        u8                        lmc;
        u8                        max_vl_num;
        u8                        sm_sl;
        u8                        subnet_timeout;
        u8                        init_type_reply;
        u8                        active_width;
        u16                        active_speed;
        u8                      phys_state;
        u16                        port_cap_flags2;
};

enum ib_device_modify_flags {
        IB_DEVICE_MODIFY_SYS_IMAGE_GUID        = 1 << 0,
        IB_DEVICE_MODIFY_NODE_DESC        = 1 << 1
};

#define IB_DEVICE_NODE_DESC_MAX 64

struct ib_device_modify {
        u64        sys_image_guid;
        char        node_desc[IB_DEVICE_NODE_DESC_MAX];
};

enum ib_port_modify_flags {
        IB_PORT_SHUTDOWN                = 1,
        IB_PORT_INIT_TYPE                = (1<<2),
        IB_PORT_RESET_QKEY_CNTR                = (1<<3),
        IB_PORT_OPA_MASK_CHG                = (1<<4)
};

struct ib_port_modify {
        u32        set_port_cap_mask;
        u32        clr_port_cap_mask;
        u8        init_type;
};

enum ib_event_type {
        IB_EVENT_CQ_ERR,
        IB_EVENT_QP_FATAL,
        IB_EVENT_QP_REQ_ERR,
        IB_EVENT_QP_ACCESS_ERR,
        IB_EVENT_COMM_EST,
        IB_EVENT_SQ_DRAINED,
        IB_EVENT_PATH_MIG,
        IB_EVENT_PATH_MIG_ERR,
        IB_EVENT_DEVICE_FATAL,
        IB_EVENT_PORT_ACTIVE,
        IB_EVENT_PORT_ERR,
        IB_EVENT_LID_CHANGE,
        IB_EVENT_PKEY_CHANGE,
        IB_EVENT_SM_CHANGE,
        IB_EVENT_SRQ_ERR,
        IB_EVENT_SRQ_LIMIT_REACHED,
        IB_EVENT_QP_LAST_WQE_REACHED,
        IB_EVENT_CLIENT_REREGISTER,
        IB_EVENT_GID_CHANGE,
        IB_EVENT_WQ_FATAL,
};

const char *__attribute_const__ ib_event_msg(enum ib_event_type event);

struct ib_event {
        struct ib_device        *device;
        union {
                struct ib_cq        *cq;
                struct ib_qp        *qp;
                struct ib_srq        *srq;
                struct ib_wq        *wq;
                u32                port_num;
        } element;
        enum ib_event_type        event;
};

struct ib_event_handler {
        struct ib_device *device;
        void            (*handler)(struct ib_event_handler *, struct ib_event *);
        struct list_head  list;
};

#define INIT_IB_EVENT_HANDLER(_ptr, _device, _handler)                \
        do {                                                        \
                (_ptr)->device  = _device;                        \
                (_ptr)->handler = _handler;                        \
                INIT_LIST_HEAD(&(_ptr)->list);                        \
        } while (0)

struct ib_global_route {
        const struct ib_gid_attr *sgid_attr;
        union ib_gid        dgid;
        u32                flow_label;
        u8                sgid_index;
        u8                hop_limit;
        u8                traffic_class;
};

struct ib_grh {
        __be32                version_tclass_flow;
        __be16                paylen;
        u8                next_hdr;
        u8                hop_limit;
        union ib_gid        sgid;
        union ib_gid        dgid;
};

union rdma_network_hdr {
        struct ib_grh ibgrh;
        struct {
                /* The IB spec states that if it's IPv4, the header
                 * is located in the last 20 bytes of the header.
                 */
                u8                reserved[20];
                struct iphdr        roce4grh;
        };
};

#define IB_QPN_MASK                0xFFFFFF

enum {
        IB_MULTICAST_QPN = 0xffffff
};

#define IB_LID_PERMISSIVE        cpu_to_be16(0xFFFF)
#define IB_MULTICAST_LID_BASE        cpu_to_be16(0xC000)

enum ib_ah_flags {
        IB_AH_GRH        = 1
};

enum ib_rate {
        IB_RATE_PORT_CURRENT = 0,
        IB_RATE_2_5_GBPS = 2,
        IB_RATE_5_GBPS   = 5,
        IB_RATE_10_GBPS  = 3,
        IB_RATE_20_GBPS  = 6,
        IB_RATE_30_GBPS  = 4,
        IB_RATE_40_GBPS  = 7,
        IB_RATE_60_GBPS  = 8,
        IB_RATE_80_GBPS  = 9,
        IB_RATE_120_GBPS = 10,
        IB_RATE_14_GBPS  = 11,
        IB_RATE_56_GBPS  = 12,
        IB_RATE_112_GBPS = 13,
        IB_RATE_168_GBPS = 14,
        IB_RATE_25_GBPS  = 15,
        IB_RATE_100_GBPS = 16,
        IB_RATE_200_GBPS = 17,
        IB_RATE_300_GBPS = 18,
        IB_RATE_28_GBPS  = 19,
        IB_RATE_50_GBPS  = 20,
        IB_RATE_400_GBPS = 21,
        IB_RATE_600_GBPS = 22,
        IB_RATE_800_GBPS = 23,
};

/**
 * ib_rate_to_mult - Convert the IB rate enum to a multiple of the
 * base rate of 2.5 Gbit/sec.  For example, IB_RATE_5_GBPS will be
 * converted to 2, since 5 Gbit/sec is 2 * 2.5 Gbit/sec.
 * @rate: rate to convert.
 */
__attribute_const__ int ib_rate_to_mult(enum ib_rate rate);

/**
 * ib_rate_to_mbps - Convert the IB rate enum to Mbps.
 * For example, IB_RATE_2_5_GBPS will be converted to 2500.
 * @rate: rate to convert.
 */
__attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);


/**
 * enum ib_mr_type - memory region type
 * @IB_MR_TYPE_MEM_REG:       memory region that is used for
 *                            normal registration
 * @IB_MR_TYPE_SG_GAPS:       memory region that is capable to
 *                            register any arbitrary sg lists (without
 *                            the normal mr constraints - see
 *                            ib_map_mr_sg)
 * @IB_MR_TYPE_DM:            memory region that is used for device
 *                            memory registration
 * @IB_MR_TYPE_USER:          memory region that is used for the user-space
 *                            application
 * @IB_MR_TYPE_DMA:           memory region that is used for DMA operations
 *                            without address translations (VA=PA)
 * @IB_MR_TYPE_INTEGRITY:     memory region that is used for
 *                            data integrity operations
 */
enum ib_mr_type {
        IB_MR_TYPE_MEM_REG,
        IB_MR_TYPE_SG_GAPS,
        IB_MR_TYPE_DM,
        IB_MR_TYPE_USER,
        IB_MR_TYPE_DMA,
        IB_MR_TYPE_INTEGRITY,
};

enum ib_mr_status_check {
        IB_MR_CHECK_SIG_STATUS = 1,
};

/**
 * struct ib_mr_status - Memory region status container
 *
 * @fail_status: Bitmask of MR checks status. For each
 *     failed check a corresponding status bit is set.
 * @sig_err: Additional info for IB_MR_CEHCK_SIG_STATUS
 *     failure.
 */
struct ib_mr_status {
        u32                    fail_status;
        struct ib_sig_err   sig_err;
};

/**
 * mult_to_ib_rate - Convert a multiple of 2.5 Gbit/sec to an IB rate
 * enum.
 * @mult: multiple to convert.
 */
__attribute_const__ enum ib_rate mult_to_ib_rate(int mult);

struct rdma_ah_init_attr {
        struct rdma_ah_attr *ah_attr;
        u32 flags;
        struct net_device *xmit_slave;
};

enum rdma_ah_attr_type {
        RDMA_AH_ATTR_TYPE_UNDEFINED,
        RDMA_AH_ATTR_TYPE_IB,
        RDMA_AH_ATTR_TYPE_ROCE,
        RDMA_AH_ATTR_TYPE_OPA,
};

struct ib_ah_attr {
        u16                        dlid;
        u8                        src_path_bits;
};

struct roce_ah_attr {
        u8                        dmac[ETH_ALEN];
};

struct opa_ah_attr {
        u32                        dlid;
        u8                        src_path_bits;
        bool                        make_grd;
};

struct rdma_ah_attr {
        struct ib_global_route        grh;
        u8                        sl;
        u8                        static_rate;
        u32                        port_num;
        u8                        ah_flags;
        enum rdma_ah_attr_type type;
        union {
                struct ib_ah_attr ib;
                struct roce_ah_attr roce;
                struct opa_ah_attr opa;
        };
};

enum ib_wc_status {
        IB_WC_SUCCESS,
        IB_WC_LOC_LEN_ERR,
        IB_WC_LOC_QP_OP_ERR,
        IB_WC_LOC_EEC_OP_ERR,
        IB_WC_LOC_PROT_ERR,
        IB_WC_WR_FLUSH_ERR,
        IB_WC_MW_BIND_ERR,
        IB_WC_BAD_RESP_ERR,
        IB_WC_LOC_ACCESS_ERR,
        IB_WC_REM_INV_REQ_ERR,
        IB_WC_REM_ACCESS_ERR,
        IB_WC_REM_OP_ERR,
        IB_WC_RETRY_EXC_ERR,
        IB_WC_RNR_RETRY_EXC_ERR,
        IB_WC_LOC_RDD_VIOL_ERR,
        IB_WC_REM_INV_RD_REQ_ERR,
        IB_WC_REM_ABORT_ERR,
        IB_WC_INV_EECN_ERR,
        IB_WC_INV_EEC_STATE_ERR,
        IB_WC_FATAL_ERR,
        IB_WC_RESP_TIMEOUT_ERR,
        IB_WC_GENERAL_ERR
};

const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status);

enum ib_wc_opcode {
        IB_WC_SEND = IB_UVERBS_WC_SEND,
        IB_WC_RDMA_WRITE = IB_UVERBS_WC_RDMA_WRITE,
        IB_WC_RDMA_READ = IB_UVERBS_WC_RDMA_READ,
        IB_WC_COMP_SWAP = IB_UVERBS_WC_COMP_SWAP,
        IB_WC_FETCH_ADD = IB_UVERBS_WC_FETCH_ADD,
        IB_WC_BIND_MW = IB_UVERBS_WC_BIND_MW,
        IB_WC_LOCAL_INV = IB_UVERBS_WC_LOCAL_INV,
        IB_WC_LSO = IB_UVERBS_WC_TSO,
        IB_WC_ATOMIC_WRITE = IB_UVERBS_WC_ATOMIC_WRITE,
        IB_WC_REG_MR,
        IB_WC_MASKED_COMP_SWAP,
        IB_WC_MASKED_FETCH_ADD,
        IB_WC_FLUSH = IB_UVERBS_WC_FLUSH,
/*
 * Set value of IB_WC_RECV so consumers can test if a completion is a
 * receive by testing (opcode & IB_WC_RECV).
 */
        IB_WC_RECV                        = 1 << 7,
        IB_WC_RECV_RDMA_WITH_IMM
};

enum ib_wc_flags {
        IB_WC_GRH                = 1,
        IB_WC_WITH_IMM                = (1<<1),
        IB_WC_WITH_INVALIDATE        = (1<<2),
        IB_WC_IP_CSUM_OK        = (1<<3),
        IB_WC_WITH_SMAC                = (1<<4),
        IB_WC_WITH_VLAN                = (1<<5),
        IB_WC_WITH_NETWORK_HDR_TYPE        = (1<<6),
};

struct ib_wc {
        union {
                u64                wr_id;
                struct ib_cqe        *wr_cqe;
        };
        enum ib_wc_status        status;
        enum ib_wc_opcode        opcode;
        u32                        vendor_err;
        u32                        byte_len;
        struct ib_qp               *qp;
        union {
                __be32                imm_data;
                u32                invalidate_rkey;
        } ex;
        u32                        src_qp;
        u32                        slid;
        int                        wc_flags;
        u16                        pkey_index;
        u8                        sl;
        u8                        dlid_path_bits;
        u32 port_num; /* valid only for DR SMPs on switches */
        u8                        smac[ETH_ALEN];
        u16                        vlan_id;
        u8                        network_hdr_type;
};

enum ib_cq_notify_flags {
        IB_CQ_SOLICITED                        = 1 << 0,
        IB_CQ_NEXT_COMP                        = 1 << 1,
        IB_CQ_SOLICITED_MASK                = IB_CQ_SOLICITED | IB_CQ_NEXT_COMP,
        IB_CQ_REPORT_MISSED_EVENTS        = 1 << 2,
};

enum ib_srq_type {
        IB_SRQT_BASIC = IB_UVERBS_SRQT_BASIC,
        IB_SRQT_XRC = IB_UVERBS_SRQT_XRC,
        IB_SRQT_TM = IB_UVERBS_SRQT_TM,
};

static inline bool ib_srq_has_cq(enum ib_srq_type srq_type)
{
        return srq_type == IB_SRQT_XRC ||
               srq_type == IB_SRQT_TM;
}

enum ib_srq_attr_mask {
        IB_SRQ_MAX_WR        = 1 << 0,
        IB_SRQ_LIMIT        = 1 << 1,
};

struct ib_srq_attr {
        u32        max_wr;
        u32        max_sge;
        u32        srq_limit;
};

struct ib_srq_init_attr {
        void                      (*event_handler)(struct ib_event *, void *);
        void                       *srq_context;
        struct ib_srq_attr        attr;
        enum ib_srq_type        srq_type;

        struct {
                struct ib_cq   *cq;
                union {
                        struct {
                                struct ib_xrcd *xrcd;
                        } xrc;

                        struct {
                                u32                max_num_tags;
                        } tag_matching;
                };
        } ext;
};

struct ib_qp_cap {
        u32        max_send_wr;
        u32        max_recv_wr;
        u32        max_send_sge;
        u32        max_recv_sge;
        u32        max_inline_data;

        /*
         * Maximum number of rdma_rw_ctx structures in flight at a time.
         * ib_create_qp() will calculate the right amount of needed WRs
         * and MRs based on this.
         */
        u32        max_rdma_ctxs;
};

enum ib_sig_type {
        IB_SIGNAL_ALL_WR,
        IB_SIGNAL_REQ_WR
};

enum ib_qp_type {
        /*
         * IB_QPT_SMI and IB_QPT_GSI have to be the first two entries
         * here (and in that order) since the MAD layer uses them as
         * indices into a 2-entry table.
         */
        IB_QPT_SMI,
        IB_QPT_GSI,

        IB_QPT_RC = IB_UVERBS_QPT_RC,
        IB_QPT_UC = IB_UVERBS_QPT_UC,
        IB_QPT_UD = IB_UVERBS_QPT_UD,
        IB_QPT_RAW_IPV6,
        IB_QPT_RAW_ETHERTYPE,
        IB_QPT_RAW_PACKET = IB_UVERBS_QPT_RAW_PACKET,
        IB_QPT_XRC_INI = IB_UVERBS_QPT_XRC_INI,
        IB_QPT_XRC_TGT = IB_UVERBS_QPT_XRC_TGT,
        IB_QPT_MAX,
        IB_QPT_DRIVER = IB_UVERBS_QPT_DRIVER,
        /* Reserve a range for qp types internal to the low level driver.
         * These qp types will not be visible at the IB core layer, so the
         * IB_QPT_MAX usages should not be affected in the core layer
         */
        IB_QPT_RESERVED1 = 0x1000,
        IB_QPT_RESERVED2,
        IB_QPT_RESERVED3,
        IB_QPT_RESERVED4,
        IB_QPT_RESERVED5,
        IB_QPT_RESERVED6,
        IB_QPT_RESERVED7,
        IB_QPT_RESERVED8,
        IB_QPT_RESERVED9,
        IB_QPT_RESERVED10,
};

enum ib_qp_create_flags {
        IB_QP_CREATE_IPOIB_UD_LSO                = 1 << 0,
        IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK        =
                IB_UVERBS_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
        IB_QP_CREATE_CROSS_CHANNEL              = 1 << 2,
        IB_QP_CREATE_MANAGED_SEND               = 1 << 3,
        IB_QP_CREATE_MANAGED_RECV               = 1 << 4,
        IB_QP_CREATE_NETIF_QP                        = 1 << 5,
        IB_QP_CREATE_INTEGRITY_EN                = 1 << 6,
        IB_QP_CREATE_NETDEV_USE                        = 1 << 7,
        IB_QP_CREATE_SCATTER_FCS                =
                IB_UVERBS_QP_CREATE_SCATTER_FCS,
        IB_QP_CREATE_CVLAN_STRIPPING                =
                IB_UVERBS_QP_CREATE_CVLAN_STRIPPING,
        IB_QP_CREATE_SOURCE_QPN                        = 1 << 10,
        IB_QP_CREATE_PCI_WRITE_END_PADDING        =
                IB_UVERBS_QP_CREATE_PCI_WRITE_END_PADDING,
        /* reserve bits 26-31 for low level drivers' internal use */
        IB_QP_CREATE_RESERVED_START                = 1 << 26,
        IB_QP_CREATE_RESERVED_END                = 1 << 31,
};

/*
 * Note: users may not call ib_close_qp or ib_destroy_qp from the event_handler
 * callback to destroy the passed in QP.
 */

struct ib_qp_init_attr {
        /* This callback occurs in workqueue context */
        void                  (*event_handler)(struct ib_event *, void *);

        void                       *qp_context;
        struct ib_cq               *send_cq;
        struct ib_cq               *recv_cq;
        struct ib_srq               *srq;
        struct ib_xrcd               *xrcd;     /* XRC TGT QPs only */
        struct ib_qp_cap        cap;
        enum ib_sig_type        sq_sig_type;
        enum ib_qp_type                qp_type;
        u32                        create_flags;

        /*
         * Only needed for special QP types, or when using the RW API.
         */
        u32                        port_num;
        struct ib_rwq_ind_table *rwq_ind_tbl;
        u32                        source_qpn;
};

struct ib_qp_open_attr {
        void                  (*event_handler)(struct ib_event *, void *);
        void                       *qp_context;
        u32                        qp_num;
        enum ib_qp_type                qp_type;
};

enum ib_rnr_timeout {
        IB_RNR_TIMER_655_36 =  0,
        IB_RNR_TIMER_000_01 =  1,
        IB_RNR_TIMER_000_02 =  2,
        IB_RNR_TIMER_000_03 =  3,
        IB_RNR_TIMER_000_04 =  4,
        IB_RNR_TIMER_000_06 =  5,
        IB_RNR_TIMER_000_08 =  6,
        IB_RNR_TIMER_000_12 =  7,
        IB_RNR_TIMER_000_16 =  8,
        IB_RNR_TIMER_000_24 =  9,
        IB_RNR_TIMER_000_32 = 10,
        IB_RNR_TIMER_000_48 = 11,
        IB_RNR_TIMER_000_64 = 12,
        IB_RNR_TIMER_000_96 = 13,
        IB_RNR_TIMER_001_28 = 14,
        IB_RNR_TIMER_001_92 = 15,
        IB_RNR_TIMER_002_56 = 16,
        IB_RNR_TIMER_003_84 = 17,
        IB_RNR_TIMER_005_12 = 18,
        IB_RNR_TIMER_007_68 = 19,
        IB_RNR_TIMER_010_24 = 20,
        IB_RNR_TIMER_015_36 = 21,
        IB_RNR_TIMER_020_48 = 22,
        IB_RNR_TIMER_030_72 = 23,
        IB_RNR_TIMER_040_96 = 24,
        IB_RNR_TIMER_061_44 = 25,
        IB_RNR_TIMER_081_92 = 26,
        IB_RNR_TIMER_122_88 = 27,
        IB_RNR_TIMER_163_84 = 28,
        IB_RNR_TIMER_245_76 = 29,
        IB_RNR_TIMER_327_68 = 30,
        IB_RNR_TIMER_491_52 = 31
};

enum ib_qp_attr_mask {
        IB_QP_STATE                        = 1,
        IB_QP_CUR_STATE                        = (1<<1),
        IB_QP_EN_SQD_ASYNC_NOTIFY        = (1<<2),
        IB_QP_ACCESS_FLAGS                = (1<<3),
        IB_QP_PKEY_INDEX                = (1<<4),
        IB_QP_PORT                        = (1<<5),
        IB_QP_QKEY                        = (1<<6),
        IB_QP_AV                        = (1<<7),
        IB_QP_PATH_MTU                        = (1<<8),
        IB_QP_TIMEOUT                        = (1<<9),
        IB_QP_RETRY_CNT                        = (1<<10),
        IB_QP_RNR_RETRY                        = (1<<11),
        IB_QP_RQ_PSN                        = (1<<12),
        IB_QP_MAX_QP_RD_ATOMIC                = (1<<13),
        IB_QP_ALT_PATH                        = (1<<14),
        IB_QP_MIN_RNR_TIMER                = (1<<15),
        IB_QP_SQ_PSN                        = (1<<16),
        IB_QP_MAX_DEST_RD_ATOMIC        = (1<<17),
        IB_QP_PATH_MIG_STATE                = (1<<18),
        IB_QP_CAP                        = (1<<19),
        IB_QP_DEST_QPN                        = (1<<20),
        IB_QP_RESERVED1                        = (1<<21),
        IB_QP_RESERVED2                        = (1<<22),
        IB_QP_RESERVED3                        = (1<<23),
        IB_QP_RESERVED4                        = (1<<24),
        IB_QP_RATE_LIMIT                = (1<<25),

        IB_QP_ATTR_STANDARD_BITS = GENMASK(20, 0),
};

enum ib_qp_state {
        IB_QPS_RESET,
        IB_QPS_INIT,
        IB_QPS_RTR,
        IB_QPS_RTS,
        IB_QPS_SQD,
        IB_QPS_SQE,
        IB_QPS_ERR
};

enum ib_mig_state {
        IB_MIG_MIGRATED,
        IB_MIG_REARM,
        IB_MIG_ARMED
};

enum ib_mw_type {
        IB_MW_TYPE_1 = 1,
        IB_MW_TYPE_2 = 2
};

struct ib_qp_attr {
        enum ib_qp_state        qp_state;
        enum ib_qp_state        cur_qp_state;
        enum ib_mtu                path_mtu;
        enum ib_mig_state        path_mig_state;
        u32                        qkey;
        u32                        rq_psn;
        u32                        sq_psn;
        u32                        dest_qp_num;
        int                        qp_access_flags;
        struct ib_qp_cap        cap;
        struct rdma_ah_attr        ah_attr;
        struct rdma_ah_attr        alt_ah_attr;
        u16                        pkey_index;
        u16                        alt_pkey_index;
        u8                        en_sqd_async_notify;
        u8                        sq_draining;
        u8                        max_rd_atomic;
        u8                        max_dest_rd_atomic;
        u8                        min_rnr_timer;
        u32                        port_num;
        u8                        timeout;
        u8                        retry_cnt;
        u8                        rnr_retry;
        u32                        alt_port_num;
        u8                        alt_timeout;
        u32                        rate_limit;
        struct net_device        *xmit_slave;
};

enum ib_wr_opcode {
        /* These are shared with userspace */
        IB_WR_RDMA_WRITE = IB_UVERBS_WR_RDMA_WRITE,
        IB_WR_RDMA_WRITE_WITH_IMM = IB_UVERBS_WR_RDMA_WRITE_WITH_IMM,
        IB_WR_SEND = IB_UVERBS_WR_SEND,
        IB_WR_SEND_WITH_IMM = IB_UVERBS_WR_SEND_WITH_IMM,
        IB_WR_RDMA_READ = IB_UVERBS_WR_RDMA_READ,
        IB_WR_ATOMIC_CMP_AND_SWP = IB_UVERBS_WR_ATOMIC_CMP_AND_SWP,
        IB_WR_ATOMIC_FETCH_AND_ADD = IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD,
        IB_WR_BIND_MW = IB_UVERBS_WR_BIND_MW,
        IB_WR_LSO = IB_UVERBS_WR_TSO,
        IB_WR_SEND_WITH_INV = IB_UVERBS_WR_SEND_WITH_INV,
        IB_WR_RDMA_READ_WITH_INV = IB_UVERBS_WR_RDMA_READ_WITH_INV,
        IB_WR_LOCAL_INV = IB_UVERBS_WR_LOCAL_INV,
        IB_WR_MASKED_ATOMIC_CMP_AND_SWP =
                IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP,
        IB_WR_MASKED_ATOMIC_FETCH_AND_ADD =
                IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD,
        IB_WR_FLUSH = IB_UVERBS_WR_FLUSH,
        IB_WR_ATOMIC_WRITE = IB_UVERBS_WR_ATOMIC_WRITE,

        /* These are kernel only and can not be issued by userspace */
        IB_WR_REG_MR = 0x20,
        IB_WR_REG_MR_INTEGRITY,

        /* reserve values for low level drivers' internal use.
         * These values will not be used at all in the ib core layer.
         */
        IB_WR_RESERVED1 = 0xf0,
        IB_WR_RESERVED2,
        IB_WR_RESERVED3,
        IB_WR_RESERVED4,
        IB_WR_RESERVED5,
        IB_WR_RESERVED6,
        IB_WR_RESERVED7,
        IB_WR_RESERVED8,
        IB_WR_RESERVED9,
        IB_WR_RESERVED10,
};

enum ib_send_flags {
        IB_SEND_FENCE                = 1,
        IB_SEND_SIGNALED        = (1<<1),
        IB_SEND_SOLICITED        = (1<<2),
        IB_SEND_INLINE                = (1<<3),
        IB_SEND_IP_CSUM                = (1<<4),

        /* reserve bits 26-31 for low level drivers' internal use */
        IB_SEND_RESERVED_START        = (1 << 26),
        IB_SEND_RESERVED_END        = (1 << 31),
};

struct ib_sge {
        u64        addr;
        u32        length;
        u32        lkey;
};

struct ib_cqe {
        void (*done)(struct ib_cq *cq, struct ib_wc *wc);
};

struct ib_send_wr {
        struct ib_send_wr      *next;
        union {
                u64                wr_id;
                struct ib_cqe        *wr_cqe;
        };
        struct ib_sge               *sg_list;
        int                        num_sge;
        enum ib_wr_opcode        opcode;
        int                        send_flags;
        union {
                __be32                imm_data;
                u32                invalidate_rkey;
        } ex;
};

struct ib_rdma_wr {
        struct ib_send_wr        wr;
        u64                        remote_addr;
        u32                        rkey;
};

static inline const struct ib_rdma_wr *rdma_wr(const struct ib_send_wr *wr)
{
        return container_of(wr, struct ib_rdma_wr, wr);
}

struct ib_atomic_wr {
        struct ib_send_wr        wr;
        u64                        remote_addr;
        u64                        compare_add;
        u64                        swap;
        u64                        compare_add_mask;
        u64                        swap_mask;
        u32                        rkey;
};

static inline const struct ib_atomic_wr *atomic_wr(const struct ib_send_wr *wr)
{
        return container_of(wr, struct ib_atomic_wr, wr);
}

struct ib_ud_wr {
        struct ib_send_wr        wr;
        struct ib_ah                *ah;
        void                        *header;
        int                        hlen;
        int                        mss;
        u32                        remote_qpn;
        u32                        remote_qkey;
        u16                        pkey_index; /* valid for GSI only */
        u32                        port_num; /* valid for DR SMPs on switch only */
};

static inline const struct ib_ud_wr *ud_wr(const struct ib_send_wr *wr)
{
        return container_of(wr, struct ib_ud_wr, wr);
}

struct ib_reg_wr {
        struct ib_send_wr        wr;
        struct ib_mr                *mr;
        u32                        key;
        int                        access;
};

static inline const struct ib_reg_wr *reg_wr(const struct ib_send_wr *wr)
{
        return container_of(wr, struct ib_reg_wr, wr);
}

struct ib_recv_wr {
        struct ib_recv_wr      *next;
        union {
                u64                wr_id;
                struct ib_cqe        *wr_cqe;
        };
        struct ib_sge               *sg_list;
        int                        num_sge;
};

enum ib_access_flags {
        IB_ACCESS_LOCAL_WRITE = IB_UVERBS_ACCESS_LOCAL_WRITE,
        IB_ACCESS_REMOTE_WRITE = IB_UVERBS_ACCESS_REMOTE_WRITE,
        IB_ACCESS_REMOTE_READ = IB_UVERBS_ACCESS_REMOTE_READ,
        IB_ACCESS_REMOTE_ATOMIC = IB_UVERBS_ACCESS_REMOTE_ATOMIC,
        IB_ACCESS_MW_BIND = IB_UVERBS_ACCESS_MW_BIND,
        IB_ZERO_BASED = IB_UVERBS_ACCESS_ZERO_BASED,
        IB_ACCESS_ON_DEMAND = IB_UVERBS_ACCESS_ON_DEMAND,
        IB_ACCESS_HUGETLB = IB_UVERBS_ACCESS_HUGETLB,
        IB_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_RELAXED_ORDERING,
        IB_ACCESS_FLUSH_GLOBAL = IB_UVERBS_ACCESS_FLUSH_GLOBAL,
        IB_ACCESS_FLUSH_PERSISTENT = IB_UVERBS_ACCESS_FLUSH_PERSISTENT,

        IB_ACCESS_OPTIONAL = IB_UVERBS_ACCESS_OPTIONAL_RANGE,
        IB_ACCESS_SUPPORTED =
                ((IB_ACCESS_FLUSH_PERSISTENT << 1) - 1) | IB_ACCESS_OPTIONAL,
};

/*
 * XXX: these are apparently used for ->rereg_user_mr, no idea why they
 * are hidden here instead of a uapi header!
 */
enum ib_mr_rereg_flags {
        IB_MR_REREG_TRANS        = 1,
        IB_MR_REREG_PD                = (1<<1),
        IB_MR_REREG_ACCESS        = (1<<2),
        IB_MR_REREG_SUPPORTED        = ((IB_MR_REREG_ACCESS << 1) - 1)
};

struct ib_umem;

enum rdma_remove_reason {
        /*
         * Userspace requested uobject deletion or initial try
         * to remove uobject via cleanup. Call could fail
         */
        RDMA_REMOVE_DESTROY,
        /* Context deletion. This call should delete the actual object itself */
        RDMA_REMOVE_CLOSE,
        /* Driver is being hot-unplugged. This call should delete the actual object itself */
        RDMA_REMOVE_DRIVER_REMOVE,
        /* uobj is being cleaned-up before being committed */
        RDMA_REMOVE_ABORT,
        /* The driver failed to destroy the uobject and is being disconnected */
        RDMA_REMOVE_DRIVER_FAILURE,
};

struct ib_rdmacg_object {
#ifdef CONFIG_CGROUP_RDMA
        struct rdma_cgroup        *cg;                /* owner rdma cgroup */
#endif
};

struct ib_ucontext {
        struct ib_device       *device;
        struct ib_uverbs_file  *ufile;

        struct ib_rdmacg_object        cg_obj;
        u64 enabled_caps;
        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct rdma_restrack_entry res;
        struct xarray mmap_xa;
};

struct ib_uobject {
        u64                        user_handle;        /* handle given to us by userspace */
        /* ufile & ucontext owning this object */
        struct ib_uverbs_file  *ufile;
        /* FIXME, save memory: ufile->context == context */
        struct ib_ucontext     *context;        /* associated user context */
        void                       *object;                /* containing object */
        struct list_head        list;                /* link to context's list */
        struct ib_rdmacg_object        cg_obj;                /* rdmacg object */
        int                        id;                /* index into kernel idr */
        struct kref                ref;
        atomic_t                usecnt;                /* protects exclusive access */
        struct rcu_head                rcu;                /* kfree_rcu() overhead */

        const struct uverbs_api_object *uapi_object;
};

struct ib_udata {
        const void __user *inbuf;
        void __user *outbuf;
        size_t       inlen;
        size_t       outlen;
};

struct ib_pd {
        u32                        local_dma_lkey;
        u32                        flags;
        struct ib_device       *device;
        struct ib_uobject      *uobject;
        atomic_t                  usecnt; /* count all resources */

        u32                        unsafe_global_rkey;

        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct ib_mr               *__internal_mr;
        struct rdma_restrack_entry res;
};

struct ib_xrcd {
        struct ib_device       *device;
        atomic_t                usecnt; /* count all exposed resources */
        struct inode               *inode;
        struct rw_semaphore        tgt_qps_rwsem;
        struct xarray                tgt_qps;
};

struct ib_ah {
        struct ib_device        *device;
        struct ib_pd                *pd;
        struct ib_uobject        *uobject;
        const struct ib_gid_attr *sgid_attr;
        enum rdma_ah_attr_type        type;
};

typedef void (*ib_comp_handler)(struct ib_cq *cq, void *cq_context);

enum ib_poll_context {
        IB_POLL_SOFTIRQ,           /* poll from softirq context */
        IB_POLL_WORKQUEUE,           /* poll from workqueue */
        IB_POLL_UNBOUND_WORKQUEUE, /* poll from unbound workqueue */
        IB_POLL_LAST_POOL_TYPE = IB_POLL_UNBOUND_WORKQUEUE,

        IB_POLL_DIRECT,                   /* caller context, no hw completions */
};

struct ib_cq {
        struct ib_device       *device;
        struct ib_ucq_object   *uobject;
        ib_comp_handler           comp_handler;
        void                  (*event_handler)(struct ib_event *, void *);
        void                   *cq_context;
        int                       cqe;
        unsigned int                cqe_used;
        atomic_t                  usecnt; /* count number of work queues */
        enum ib_poll_context        poll_ctx;
        struct ib_wc                *wc;
        struct list_head        pool_entry;
        union {
                struct irq_poll                iop;
                struct work_struct        work;
        };
        struct workqueue_struct *comp_wq;
        struct dim *dim;

        /* updated only by trace points */
        ktime_t timestamp;
        u8 interrupt:1;
        u8 shared:1;
        unsigned int comp_vector;

        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct rdma_restrack_entry res;
};

struct ib_srq {
        struct ib_device       *device;
        struct ib_pd               *pd;
        struct ib_usrq_object  *uobject;
        void                      (*event_handler)(struct ib_event *, void *);
        void                       *srq_context;
        enum ib_srq_type        srq_type;
        atomic_t                usecnt;

        struct {
                struct ib_cq   *cq;
                union {
                        struct {
                                struct ib_xrcd *xrcd;
                                u32                srq_num;
                        } xrc;
                };
        } ext;

        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct rdma_restrack_entry res;
};

enum ib_raw_packet_caps {
        /*
         * Strip cvlan from incoming packet and report it in the matching work
         * completion is supported.
         */
        IB_RAW_PACKET_CAP_CVLAN_STRIPPING =
                IB_UVERBS_RAW_PACKET_CAP_CVLAN_STRIPPING,
        /*
         * Scatter FCS field of an incoming packet to host memory is supported.
         */
        IB_RAW_PACKET_CAP_SCATTER_FCS = IB_UVERBS_RAW_PACKET_CAP_SCATTER_FCS,
        /* Checksum offloads are supported (for both send and receive). */
        IB_RAW_PACKET_CAP_IP_CSUM = IB_UVERBS_RAW_PACKET_CAP_IP_CSUM,
        /*
         * When a packet is received for an RQ with no receive WQEs, the
         * packet processing is delayed.
         */
        IB_RAW_PACKET_CAP_DELAY_DROP = IB_UVERBS_RAW_PACKET_CAP_DELAY_DROP,
};

enum ib_wq_type {
        IB_WQT_RQ = IB_UVERBS_WQT_RQ,
};

enum ib_wq_state {
        IB_WQS_RESET,
        IB_WQS_RDY,
        IB_WQS_ERR
};

struct ib_wq {
        struct ib_device       *device;
        struct ib_uwq_object   *uobject;
        void                    *wq_context;
        void                    (*event_handler)(struct ib_event *, void *);
        struct ib_pd               *pd;
        struct ib_cq               *cq;
        u32                wq_num;
        enum ib_wq_state       state;
        enum ib_wq_type        wq_type;
        atomic_t                usecnt;
};

enum ib_wq_flags {
        IB_WQ_FLAGS_CVLAN_STRIPPING        = IB_UVERBS_WQ_FLAGS_CVLAN_STRIPPING,
        IB_WQ_FLAGS_SCATTER_FCS                = IB_UVERBS_WQ_FLAGS_SCATTER_FCS,
        IB_WQ_FLAGS_DELAY_DROP                = IB_UVERBS_WQ_FLAGS_DELAY_DROP,
        IB_WQ_FLAGS_PCI_WRITE_END_PADDING =
                                IB_UVERBS_WQ_FLAGS_PCI_WRITE_END_PADDING,
};

struct ib_wq_init_attr {
        void                       *wq_context;
        enum ib_wq_type        wq_type;
        u32                max_wr;
        u32                max_sge;
        struct        ib_cq               *cq;
        void                    (*event_handler)(struct ib_event *, void *);
        u32                create_flags; /* Use enum ib_wq_flags */
};

enum ib_wq_attr_mask {
        IB_WQ_STATE                = 1 << 0,
        IB_WQ_CUR_STATE                = 1 << 1,
        IB_WQ_FLAGS                = 1 << 2,
};

struct ib_wq_attr {
        enum        ib_wq_state        wq_state;
        enum        ib_wq_state        curr_wq_state;
        u32                        flags; /* Use enum ib_wq_flags */
        u32                        flags_mask; /* Use enum ib_wq_flags */
};

struct ib_rwq_ind_table {
        struct ib_device        *device;
        struct ib_uobject      *uobject;
        atomic_t                usecnt;
        u32                ind_tbl_num;
        u32                log_ind_tbl_size;
        struct ib_wq        **ind_tbl;
};

struct ib_rwq_ind_table_init_attr {
        u32                log_ind_tbl_size;
        /* Each entry is a pointer to Receive Work Queue */
        struct ib_wq        **ind_tbl;
};

enum port_pkey_state {
        IB_PORT_PKEY_NOT_VALID = 0,
        IB_PORT_PKEY_VALID = 1,
        IB_PORT_PKEY_LISTED = 2,
};

struct ib_qp_security;

struct ib_port_pkey {
        enum port_pkey_state        state;
        u16                        pkey_index;
        u32                        port_num;
        struct list_head        qp_list;
        struct list_head        to_error_list;
        struct ib_qp_security  *sec;
};

struct ib_ports_pkeys {
        struct ib_port_pkey        main;
        struct ib_port_pkey        alt;
};

struct ib_qp_security {
        struct ib_qp               *qp;
        struct ib_device       *dev;
        /* Hold this mutex when changing port and pkey settings. */
        struct mutex                mutex;
        struct ib_ports_pkeys  *ports_pkeys;
        /* A list of all open shared QP handles.  Required to enforce security
         * properly for all users of a shared QP.
         */
        struct list_head        shared_qp_list;
        void                   *security;
        bool                        destroying;
        atomic_t                error_list_count;
        struct completion        error_complete;
        int                        error_comps_pending;
};

/*
 * @max_write_sge: Maximum SGE elements per RDMA WRITE request.
 * @max_read_sge:  Maximum SGE elements per RDMA READ request.
 */
struct ib_qp {
        struct ib_device       *device;
        struct ib_pd               *pd;
        struct ib_cq               *send_cq;
        struct ib_cq               *recv_cq;
        spinlock_t                mr_lock;
        int                        mrs_used;
        struct list_head        rdma_mrs;
        struct list_head        sig_mrs;
        struct ib_srq               *srq;
        struct completion        srq_completion;
        struct ib_xrcd               *xrcd; /* XRC TGT QPs only */
        struct list_head        xrcd_list;

        /* count times opened, mcast attaches, flow attaches */
        atomic_t                usecnt;
        struct list_head        open_list;
        struct ib_qp           *real_qp;
        struct ib_uqp_object   *uobject;
        void                  (*event_handler)(struct ib_event *, void *);
        void                  (*registered_event_handler)(struct ib_event *, void *);
        void                       *qp_context;
        /* sgid_attrs associated with the AV's */
        const struct ib_gid_attr *av_sgid_attr;
        const struct ib_gid_attr *alt_path_sgid_attr;
        u32                        qp_num;
        u32                        max_write_sge;
        u32                        max_read_sge;
        enum ib_qp_type                qp_type;
        struct ib_rwq_ind_table *rwq_ind_tbl;
        struct ib_qp_security  *qp_sec;
        u32                        port;

        bool                        integrity_en;
        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct rdma_restrack_entry     res;

        /* The counter the qp is bind to */
        struct rdma_counter    *counter;
};

struct ib_dm {
        struct ib_device  *device;
        u32                   length;
        u32                   flags;
        struct ib_uobject *uobject;
        atomic_t           usecnt;
};

struct ib_mr {
        struct ib_device  *device;
        struct ib_pd          *pd;
        u32                   lkey;
        u32                   rkey;
        u64                   iova;
        u64                   length;
        unsigned int           page_size;
        enum ib_mr_type           type;
        bool                   need_inval;
        union {
                struct ib_uobject        *uobject;        /* user */
                struct list_head        qp_entry;        /* FR */
        };

        struct ib_dm      *dm;
        struct ib_sig_attrs *sig_attrs; /* only for IB_MR_TYPE_INTEGRITY MRs */
        /*
         * Implementation details of the RDMA core, don't use in drivers:
         */
        struct rdma_restrack_entry res;
};

struct ib_mw {
        struct ib_device        *device;
        struct ib_pd                *pd;
        struct ib_uobject        *uobject;
        u32                        rkey;
        enum ib_mw_type         type;
};

/* Supported steering options */
enum ib_flow_attr_type {
        /* steering according to rule specifications */
        IB_FLOW_ATTR_NORMAL                = 0x0,
        /* default unicast and multicast rule -
         * receive all Eth traffic which isn't steered to any QP
         */
        IB_FLOW_ATTR_ALL_DEFAULT        = 0x1,
        /* default multicast rule -
         * receive all Eth multicast traffic which isn't steered to any QP
         */
        IB_FLOW_ATTR_MC_DEFAULT                = 0x2,
        /* sniffer rule - receive all port traffic */
        IB_FLOW_ATTR_SNIFFER                = 0x3
};

/* Supported steering header types */
enum ib_flow_spec_type {
        /* L2 headers*/
        IB_FLOW_SPEC_ETH                = 0x20,
        IB_FLOW_SPEC_IB                        = 0x22,
        /* L3 header*/
        IB_FLOW_SPEC_IPV4                = 0x30,
        IB_FLOW_SPEC_IPV6                = 0x31,
        IB_FLOW_SPEC_ESP                = 0x34,
        /* L4 headers*/
        IB_FLOW_SPEC_TCP                = 0x40,
        IB_FLOW_SPEC_UDP                = 0x41,
        IB_FLOW_SPEC_VXLAN_TUNNEL        = 0x50,
        IB_FLOW_SPEC_GRE                = 0x51,
        IB_FLOW_SPEC_MPLS                = 0x60,
        IB_FLOW_SPEC_INNER                = 0x100,
        /* Actions */
        IB_FLOW_SPEC_ACTION_TAG         = 0x1000,
        IB_FLOW_SPEC_ACTION_DROP        = 0x1001,
        IB_FLOW_SPEC_ACTION_HANDLE        = 0x1002,
        IB_FLOW_SPEC_ACTION_COUNT       = 0x1003,
};
#define IB_FLOW_SPEC_LAYER_MASK        0xF0
#define IB_FLOW_SPEC_SUPPORT_LAYERS 10

enum ib_flow_flags {
        IB_FLOW_ATTR_FLAGS_DONT_TRAP = 1UL << 1, /* Continue match, no steal */
        IB_FLOW_ATTR_FLAGS_EGRESS = 1UL << 2, /* Egress flow */
        IB_FLOW_ATTR_FLAGS_RESERVED  = 1UL << 3  /* Must be last */
};

struct ib_flow_eth_filter {
        u8        dst_mac[6];
        u8        src_mac[6];
        __be16        ether_type;
        __be16        vlan_tag;
};

struct ib_flow_spec_eth {
        u32                          type;
        u16                          size;
        struct ib_flow_eth_filter val;
        struct ib_flow_eth_filter mask;
};

struct ib_flow_ib_filter {
        __be16 dlid;
        __u8   sl;
};

struct ib_flow_spec_ib {
        u32                         type;
        u16                         size;
        struct ib_flow_ib_filter val;
        struct ib_flow_ib_filter mask;
};

/* IPv4 header flags */
enum ib_ipv4_flags {
        IB_IPV4_DONT_FRAG = 0x2, /* Don't enable packet fragmentation */
        IB_IPV4_MORE_FRAG = 0X4  /* For All fragmented packets except the
                                    last have this flag set */
};

struct ib_flow_ipv4_filter {
        __be32        src_ip;
        __be32        dst_ip;
        u8        proto;
        u8        tos;
        u8        ttl;
        u8        flags;
};

struct ib_flow_spec_ipv4 {
        u32                           type;
        u16                           size;
        struct ib_flow_ipv4_filter val;
        struct ib_flow_ipv4_filter mask;
};

struct ib_flow_ipv6_filter {
        u8        src_ip[16];
        u8        dst_ip[16];
        __be32        flow_label;
        u8        next_hdr;
        u8        traffic_class;
        u8        hop_limit;
} __packed;

struct ib_flow_spec_ipv6 {
        u32                           type;
        u16                           size;
        struct ib_flow_ipv6_filter val;
        struct ib_flow_ipv6_filter mask;
};

struct ib_flow_tcp_udp_filter {
        __be16        dst_port;
        __be16        src_port;
};

struct ib_flow_spec_tcp_udp {
        u32                              type;
        u16                              size;
        struct ib_flow_tcp_udp_filter val;
        struct ib_flow_tcp_udp_filter mask;
};

struct ib_flow_tunnel_filter {
        __be32        tunnel_id;
};

/* ib_flow_spec_tunnel describes the Vxlan tunnel
 * the tunnel_id from val has the vni value
 */
struct ib_flow_spec_tunnel {
        u32                              type;
        u16                              size;
        struct ib_flow_tunnel_filter  val;
        struct ib_flow_tunnel_filter  mask;
};

struct ib_flow_esp_filter {
        __be32        spi;
        __be32  seq;
};

struct ib_flow_spec_esp {
        u32                           type;
        u16                              size;
        struct ib_flow_esp_filter     val;
        struct ib_flow_esp_filter     mask;
};

struct ib_flow_gre_filter {
        __be16 c_ks_res0_ver;
        __be16 protocol;
        __be32 key;
};

struct ib_flow_spec_gre {
        u32                           type;
        u16                              size;
        struct ib_flow_gre_filter     val;
        struct ib_flow_gre_filter     mask;
};

struct ib_flow_mpls_filter {
        __be32 tag;
};

struct ib_flow_spec_mpls {
        u32                           type;
        u16                              size;
        struct ib_flow_mpls_filter     val;
        struct ib_flow_mpls_filter     mask;
};

struct ib_flow_spec_action_tag {
        enum ib_flow_spec_type              type;
        u16                              size;
        u32                           tag_id;
};

struct ib_flow_spec_action_drop {
        enum ib_flow_spec_type              type;
        u16                              size;
};

struct ib_flow_spec_action_handle {
        enum ib_flow_spec_type              type;
        u16                              size;
        struct ib_flow_action             *act;
};

enum ib_counters_description {
        IB_COUNTER_PACKETS,
        IB_COUNTER_BYTES,
};

struct ib_flow_spec_action_count {
        enum ib_flow_spec_type type;
        u16 size;
        struct ib_counters *counters;
};

union ib_flow_spec {
        struct {
                u32                        type;
                u16                        size;
        };
        struct ib_flow_spec_eth                eth;
        struct ib_flow_spec_ib                ib;
        struct ib_flow_spec_ipv4        ipv4;
        struct ib_flow_spec_tcp_udp        tcp_udp;
        struct ib_flow_spec_ipv6        ipv6;
        struct ib_flow_spec_tunnel      tunnel;
        struct ib_flow_spec_esp                esp;
        struct ib_flow_spec_gre                gre;
        struct ib_flow_spec_mpls        mpls;
        struct ib_flow_spec_action_tag  flow_tag;
        struct ib_flow_spec_action_drop drop;
        struct ib_flow_spec_action_handle action;
        struct ib_flow_spec_action_count flow_count;
};

struct ib_flow_attr {
        enum ib_flow_attr_type type;
        u16             size;
        u16             priority;
        u32             flags;
        u8             num_of_specs;
        u32             port;
        union ib_flow_spec flows[];
};

struct ib_flow {
        struct ib_qp                *qp;
        struct ib_device        *device;
        struct ib_uobject        *uobject;
};

enum ib_flow_action_type {
        IB_FLOW_ACTION_UNSPECIFIED,
        IB_FLOW_ACTION_ESP = 1,
};

struct ib_flow_action_attrs_esp_keymats {
        enum ib_uverbs_flow_action_esp_keymat                        protocol;
        union {
                struct ib_uverbs_flow_action_esp_keymat_aes_gcm aes_gcm;
        } keymat;
};

struct ib_flow_action_attrs_esp_replays {
        enum ib_uverbs_flow_action_esp_replay                        protocol;
        union {
                struct ib_uverbs_flow_action_esp_replay_bmp        bmp;
        } replay;
};

enum ib_flow_action_attrs_esp_flags {
        /* All user-space flags at the top: Use enum ib_uverbs_flow_action_esp_flags
         * This is done in order to share the same flags between user-space and
         * kernel and spare an unnecessary translation.
         */

        /* Kernel flags */
        IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED        = 1ULL << 32,
        IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS        = 1ULL << 33,
};

struct ib_flow_spec_list {
        struct ib_flow_spec_list        *next;
        union ib_flow_spec                spec;
};

struct ib_flow_action_attrs_esp {
        struct ib_flow_action_attrs_esp_keymats                *keymat;
        struct ib_flow_action_attrs_esp_replays                *replay;
        struct ib_flow_spec_list                        *encap;
        /* Used only if IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED is enabled.
         * Value of 0 is a valid value.
         */
        u32                                                esn;
        u32                                                spi;
        u32                                                seq;
        u32                                                tfc_pad;
        /* Use enum ib_flow_action_attrs_esp_flags */
        u64                                                flags;
        u64                                                hard_limit_pkts;
};

struct ib_flow_action {
        struct ib_device                *device;
        struct ib_uobject                *uobject;
        enum ib_flow_action_type        type;
        atomic_t                        usecnt;
};

struct ib_mad;

enum ib_process_mad_flags {
        IB_MAD_IGNORE_MKEY        = 1,
        IB_MAD_IGNORE_BKEY        = 2,
        IB_MAD_IGNORE_ALL        = IB_MAD_IGNORE_MKEY | IB_MAD_IGNORE_BKEY
};

enum ib_mad_result {
        IB_MAD_RESULT_FAILURE  = 0,      /* (!SUCCESS is the important flag) */
        IB_MAD_RESULT_SUCCESS  = 1 << 0, /* MAD was successfully processed   */
        IB_MAD_RESULT_REPLY    = 1 << 1, /* Reply packet needs to be sent    */
        IB_MAD_RESULT_CONSUMED = 1 << 2  /* Packet consumed: stop processing */
};

struct ib_port_cache {
        u64                      subnet_prefix;
        struct ib_pkey_cache  *pkey;
        struct ib_gid_table   *gid;
        u8                     lmc;
        enum ib_port_state     port_state;
        enum ib_port_state     last_port_state;
};

struct ib_port_immutable {
        int                           pkey_tbl_len;
        int                           gid_tbl_len;
        u32                           core_cap_flags;
        u32                           max_mad_size;
};

struct ib_port_data {
        struct ib_device *ib_dev;

        struct ib_port_immutable immutable;

        spinlock_t pkey_list_lock;

        spinlock_t netdev_lock;

        struct list_head pkey_list;

        struct ib_port_cache cache;

        struct net_device __rcu *netdev;
        netdevice_tracker netdev_tracker;
        struct hlist_node ndev_hash_link;
        struct rdma_port_counter port_counter;
        struct ib_port *sysfs;
};

/* rdma netdev type - specifies protocol type */
enum rdma_netdev_t {
        RDMA_NETDEV_OPA_VNIC,
        RDMA_NETDEV_IPOIB,
};

/**
 * struct rdma_netdev - rdma netdev
 * For cases where netstack interfacing is required.
 */
struct rdma_netdev {
        void              *clnt_priv;
        struct ib_device  *hca;
        u32                   port_num;
        int                mtu;

        /*
         * cleanup function must be specified.
         * FIXME: This is only used for OPA_VNIC and that usage should be
         * removed too.
         */
        void (*free_rdma_netdev)(struct net_device *netdev);

        /* control functions */
        void (*set_id)(struct net_device *netdev, int id);
        /* send packet */
        int (*send)(struct net_device *dev, struct sk_buff *skb,
                    struct ib_ah *address, u32 dqpn);
        /* multicast */
        int (*attach_mcast)(struct net_device *dev, struct ib_device *hca,
                            union ib_gid *gid, u16 mlid,
                            int set_qkey, u32 qkey);
        int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
                            union ib_gid *gid, u16 mlid);
        /* timeout */
        void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
};

struct rdma_netdev_alloc_params {
        size_t sizeof_priv;
        unsigned int txqs;
        unsigned int rxqs;
        void *param;

        int (*initialize_rdma_netdev)(struct ib_device *device, u32 port_num,
                                      struct net_device *netdev, void *param);
};

struct ib_odp_counters {
        atomic64_t faults;
        atomic64_t faults_handled;
        atomic64_t invalidations;
        atomic64_t invalidations_handled;
        atomic64_t prefetch;
};

struct ib_counters {
        struct ib_device        *device;
        struct ib_uobject        *uobject;
        /* num of objects attached */
        atomic_t        usecnt;
};

struct ib_counters_read_attr {
        u64        *counters_buff;
        u32        ncounters;
        u32        flags; /* use enum ib_read_counters_flags */
};

struct uverbs_attr_bundle;
struct iw_cm_id;
struct iw_cm_conn_param;

#define INIT_RDMA_OBJ_SIZE(ib_struct, drv_struct, member)                      \
        .size_##ib_struct =                                                    \
                (sizeof(struct drv_struct) +                                   \
                 BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) +      \
                 BUILD_BUG_ON_ZERO(                                            \
                         !__same_type(((struct drv_struct *)NULL)->member,     \
                                      struct ib_struct)))

#define rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, gfp)                          \
        ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
                                           gfp, false))

#define rdma_zalloc_drv_obj_numa(ib_dev, ib_type)                              \
        ((struct ib_type *)rdma_zalloc_obj(ib_dev, ib_dev->ops.size_##ib_type, \
                                           GFP_KERNEL, true))

#define rdma_zalloc_drv_obj(ib_dev, ib_type)                                   \
        rdma_zalloc_drv_obj_gfp(ib_dev, ib_type, GFP_KERNEL)

#define DECLARE_RDMA_OBJ_SIZE(ib_struct) size_t size_##ib_struct

struct rdma_user_mmap_entry {
        struct kref ref;
        struct ib_ucontext *ucontext;
        unsigned long start_pgoff;
        size_t npages;
        bool driver_removed;
};

/* Return the offset (in bytes) the user should pass to libc's mmap() */
static inline u64
rdma_user_mmap_get_offset(const struct rdma_user_mmap_entry *entry)
{
        return (u64)entry->start_pgoff << PAGE_SHIFT;
}

/**
 * struct ib_device_ops - InfiniBand device operations
 * This structure defines all the InfiniBand device operations, providers will
 * need to define the supported operations, otherwise they will be set to null.
 */
struct ib_device_ops {
        struct module *owner;
        enum rdma_driver_id driver_id;
        u32 uverbs_abi_ver;
        unsigned int uverbs_no_driver_id_binding:1;

        /*
         * NOTE: New drivers should not make use of device_group; instead new
         * device parameter should be exposed via netlink command. This
         * mechanism exists only for existing drivers.
         */
        const struct attribute_group *device_group;
        const struct attribute_group **port_groups;

        int (*post_send)(struct ib_qp *qp, const struct ib_send_wr *send_wr,
                         const struct ib_send_wr **bad_send_wr);
        int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
                         const struct ib_recv_wr **bad_recv_wr);
        void (*drain_rq)(struct ib_qp *qp);
        void (*drain_sq)(struct ib_qp *qp);
        int (*poll_cq)(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
        int (*peek_cq)(struct ib_cq *cq, int wc_cnt);
        int (*req_notify_cq)(struct ib_cq *cq, enum ib_cq_notify_flags flags);
        int (*post_srq_recv)(struct ib_srq *srq,
                             const struct ib_recv_wr *recv_wr,
                             const struct ib_recv_wr **bad_recv_wr);
        int (*process_mad)(struct ib_device *device, int process_mad_flags,
                           u32 port_num, const struct ib_wc *in_wc,
                           const struct ib_grh *in_grh,
                           const struct ib_mad *in_mad, struct ib_mad *out_mad,
                           size_t *out_mad_size, u16 *out_mad_pkey_index);
        int (*query_device)(struct ib_device *device,
                            struct ib_device_attr *device_attr,
                            struct ib_udata *udata);
        int (*modify_device)(struct ib_device *device, int device_modify_mask,
                             struct ib_device_modify *device_modify);
        void (*get_dev_fw_str)(struct ib_device *device, char *str);
        const struct cpumask *(*get_vector_affinity)(struct ib_device *ibdev,
                                                     int comp_vector);
        int (*query_port)(struct ib_device *device, u32 port_num,
                          struct ib_port_attr *port_attr);
        int (*modify_port)(struct ib_device *device, u32 port_num,
                           int port_modify_mask,
                           struct ib_port_modify *port_modify);
        /**
         * The following mandatory functions are used only at device
         * registration.  Keep functions such as these at the end of this
         * structure to avoid cache line misses when accessing struct ib_device
         * in fast paths.
         */
        int (*get_port_immutable)(struct ib_device *device, u32 port_num,
                                  struct ib_port_immutable *immutable);
        enum rdma_link_layer (*get_link_layer)(struct ib_device *device,
                                               u32 port_num);
        /**
         * When calling get_netdev, the HW vendor's driver should return the
         * net device of device @device at port @port_num or NULL if such
         * a net device doesn't exist. The vendor driver should call dev_hold
         * on this net device. The HW vendor's device driver must guarantee
         * that this function returns NULL before the net device has finished
         * NETDEV_UNREGISTER state.
         */
        struct net_device *(*get_netdev)(struct ib_device *device,
                                         u32 port_num);
        /**
         * rdma netdev operation
         *
         * Driver implementing alloc_rdma_netdev or rdma_netdev_get_params
         * must return -EOPNOTSUPP if it doesn't support the specified type.
         */
        struct net_device *(*alloc_rdma_netdev)(
                struct ib_device *device, u32 port_num, enum rdma_netdev_t type,
                const char *name, unsigned char name_assign_type,
                void (*setup)(struct net_device *));

        int (*rdma_netdev_get_params)(struct ib_device *device, u32 port_num,
                                      enum rdma_netdev_t type,
                                      struct rdma_netdev_alloc_params *params);
        /**
         * query_gid should be return GID value for @device, when @port_num
         * link layer is either IB or iWarp. It is no-op if @port_num port
         * is RoCE link layer.
         */
        int (*query_gid)(struct ib_device *device, u32 port_num, int index,
                         union ib_gid *gid);
        /**
         * When calling add_gid, the HW vendor's driver should add the gid
         * of device of port at gid index available at @attr. Meta-info of
         * that gid (for example, the network device related to this gid) is
         * available at @attr. @context allows the HW vendor driver to store
         * extra information together with a GID entry. The HW vendor driver may
         * allocate memory to contain this information and store it in @context
         * when a new GID entry is written to. Params are consistent until the
         * next call of add_gid or delete_gid. The function should return 0 on
         * success or error otherwise. The function could be called
         * concurrently for different ports. This function is only called when
         * roce_gid_table is used.
         */
        int (*add_gid)(const struct ib_gid_attr *attr, void **context);
        /**
         * When calling del_gid, the HW vendor's driver should delete the
         * gid of device @device at gid index gid_index of port port_num
         * available in @attr.
         * Upon the deletion of a GID entry, the HW vendor must free any
         * allocated memory. The caller will clear @context afterwards.
         * This function is only called when roce_gid_table is used.
         */
        int (*del_gid)(const struct ib_gid_attr *attr, void **context);
        int (*query_pkey)(struct ib_device *device, u32 port_num, u16 index,
                          u16 *pkey);
        int (*alloc_ucontext)(struct ib_ucontext *context,
                              struct ib_udata *udata);
        void (*dealloc_ucontext)(struct ib_ucontext *context);
        int (*mmap)(struct ib_ucontext *context, struct vm_area_struct *vma);
        /**
         * This will be called once refcount of an entry in mmap_xa reaches
         * zero. The type of the memory that was mapped may differ between
         * entries and is opaque to the rdma_user_mmap interface.
         * Therefore needs to be implemented by the driver in mmap_free.
         */
        void (*mmap_free)(struct rdma_user_mmap_entry *entry);
        void (*disassociate_ucontext)(struct ib_ucontext *ibcontext);
        int (*alloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
        int (*dealloc_pd)(struct ib_pd *pd, struct ib_udata *udata);
        int (*create_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
                         struct ib_udata *udata);
        int (*create_user_ah)(struct ib_ah *ah, struct rdma_ah_init_attr *attr,
                              struct ib_udata *udata);
        int (*modify_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
        int (*query_ah)(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);
        int (*destroy_ah)(struct ib_ah *ah, u32 flags);
        int (*create_srq)(struct ib_srq *srq,
                          struct ib_srq_init_attr *srq_init_attr,
                          struct ib_udata *udata);
        int (*modify_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
                          enum ib_srq_attr_mask srq_attr_mask,
                          struct ib_udata *udata);
        int (*query_srq)(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
        int (*destroy_srq)(struct ib_srq *srq, struct ib_udata *udata);
        int (*create_qp)(struct ib_qp *qp, struct ib_qp_init_attr *qp_init_attr,
                         struct ib_udata *udata);
        int (*modify_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
                         int qp_attr_mask, struct ib_udata *udata);
        int (*query_qp)(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
                        int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
        int (*destroy_qp)(struct ib_qp *qp, struct ib_udata *udata);
        int (*create_cq)(struct ib_cq *cq, const struct ib_cq_init_attr *attr,
                         struct uverbs_attr_bundle *attrs);
        int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period);
        int (*destroy_cq)(struct ib_cq *cq, struct ib_udata *udata);
        int (*resize_cq)(struct ib_cq *cq, int cqe, struct ib_udata *udata);
        struct ib_mr *(*get_dma_mr)(struct ib_pd *pd, int mr_access_flags);
        struct ib_mr *(*reg_user_mr)(struct ib_pd *pd, u64 start, u64 length,
                                     u64 virt_addr, int mr_access_flags,
                                     struct ib_udata *udata);
        struct ib_mr *(*reg_user_mr_dmabuf)(struct ib_pd *pd, u64 offset,
                                            u64 length, u64 virt_addr, int fd,
                                            int mr_access_flags,
                                            struct uverbs_attr_bundle *attrs);
        struct ib_mr *(*rereg_user_mr)(struct ib_mr *mr, int flags, u64 start,
                                       u64 length, u64 virt_addr,
                                       int mr_access_flags, struct ib_pd *pd,
                                       struct ib_udata *udata);
        int (*dereg_mr)(struct ib_mr *mr, struct ib_udata *udata);
        struct ib_mr *(*alloc_mr)(struct ib_pd *pd, enum ib_mr_type mr_type,
                                  u32 max_num_sg);
        struct ib_mr *(*alloc_mr_integrity)(struct ib_pd *pd,
                                            u32 max_num_data_sg,
                                            u32 max_num_meta_sg);
        int (*advise_mr)(struct ib_pd *pd,
                         enum ib_uverbs_advise_mr_advice advice, u32 flags,
                         struct ib_sge *sg_list, u32 num_sge,
                         struct uverbs_attr_bundle *attrs);

        /*
         * Kernel users should universally support relaxed ordering (RO), as
         * they are designed to read data only after observing the CQE and use
         * the DMA API correctly.
         *
         * Some drivers implicitly enable RO if platform supports it.
         */
        int (*map_mr_sg)(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
                         unsigned int *sg_offset);
        int (*check_mr_status)(struct ib_mr *mr, u32 check_mask,
                               struct ib_mr_status *mr_status);
        int (*alloc_mw)(struct ib_mw *mw, struct ib_udata *udata);
        int (*dealloc_mw)(struct ib_mw *mw);
        int (*attach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
        int (*detach_mcast)(struct ib_qp *qp, union ib_gid *gid, u16 lid);
        int (*alloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
        int (*dealloc_xrcd)(struct ib_xrcd *xrcd, struct ib_udata *udata);
        struct ib_flow *(*create_flow)(struct ib_qp *qp,
                                       struct ib_flow_attr *flow_attr,
                                       struct ib_udata *udata);
        int (*destroy_flow)(struct ib_flow *flow_id);
        int (*destroy_flow_action)(struct ib_flow_action *action);
        int (*set_vf_link_state)(struct ib_device *device, int vf, u32 port,
                                 int state);
        int (*get_vf_config)(struct ib_device *device, int vf, u32 port,
                             struct ifla_vf_info *ivf);
        int (*get_vf_stats)(struct ib_device *device, int vf, u32 port,
                            struct ifla_vf_stats *stats);
        int (*get_vf_guid)(struct ib_device *device, int vf, u32 port,
                            struct ifla_vf_guid *node_guid,
                            struct ifla_vf_guid *port_guid);
        int (*set_vf_guid)(struct ib_device *device, int vf, u32 port, u64 guid,
                           int type);
        struct ib_wq *(*create_wq)(struct ib_pd *pd,
                                   struct ib_wq_init_attr *init_attr,
                                   struct ib_udata *udata);
        int (*destroy_wq)(struct ib_wq *wq, struct ib_udata *udata);
        int (*modify_wq)(struct ib_wq *wq, struct ib_wq_attr *attr,
                         u32 wq_attr_mask, struct ib_udata *udata);
        int (*create_rwq_ind_table)(struct ib_rwq_ind_table *ib_rwq_ind_table,
                                    struct ib_rwq_ind_table_init_attr *init_attr,
                                    struct ib_udata *udata);
        int (*destroy_rwq_ind_table)(struct ib_rwq_ind_table *wq_ind_table);
        struct ib_dm *(*alloc_dm)(struct ib_device *device,
                                  struct ib_ucontext *context,
                                  struct ib_dm_alloc_attr *attr,
                                  struct uverbs_attr_bundle *attrs);
        int (*dealloc_dm)(struct ib_dm *dm, struct uverbs_attr_bundle *attrs);
        struct ib_mr *(*reg_dm_mr)(struct ib_pd *pd, struct ib_dm *dm,
                                   struct ib_dm_mr_attr *attr,
                                   struct uverbs_attr_bundle *attrs);
        int (*create_counters)(struct ib_counters *counters,
                               struct uverbs_attr_bundle *attrs);
        int (*destroy_counters)(struct ib_counters *counters);
        int (*read_counters)(struct ib_counters *counters,
                             struct ib_counters_read_attr *counters_read_attr,
                             struct uverbs_attr_bundle *attrs);
        int (*map_mr_sg_pi)(struct ib_mr *mr, struct scatterlist *data_sg,
                            int data_sg_nents, unsigned int *data_sg_offset,
                            struct scatterlist *meta_sg, int meta_sg_nents,
                            unsigned int *meta_sg_offset);

        /**
         * alloc_hw_[device,port]_stats - Allocate a struct rdma_hw_stats and
         *   fill in the driver initialized data.  The struct is kfree()'ed by
         *   the sysfs core when the device is removed.  A lifespan of -1 in the
         *   return struct tells the core to set a default lifespan.
         */
        struct rdma_hw_stats *(*alloc_hw_device_stats)(struct ib_device *device);
        struct rdma_hw_stats *(*alloc_hw_port_stats)(struct ib_device *device,
                                                     u32 port_num);
        /**
         * get_hw_stats - Fill in the counter value(s) in the stats struct.
         * @index - The index in the value array we wish to have updated, or
         *   num_counters if we want all stats updated
         * Return codes -
         *   < 0 - Error, no counters updated
         *   index - Updated the single counter pointed to by index
         *   num_counters - Updated all counters (will reset the timestamp
         *     and prevent further calls for lifespan milliseconds)
         * Drivers are allowed to update all counters in leiu of just the
         *   one given in index at their option
         */
        int (*get_hw_stats)(struct ib_device *device,
                            struct rdma_hw_stats *stats, u32 port, int index);

        /**
         * modify_hw_stat - Modify the counter configuration
         * @enable: true/false when enable/disable a counter
         * Return codes - 0 on success or error code otherwise.
         */
        int (*modify_hw_stat)(struct ib_device *device, u32 port,
                              unsigned int counter_index, bool enable);
        /**
         * Allows rdma drivers to add their own restrack attributes.
         */
        int (*fill_res_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);
        int (*fill_res_mr_entry_raw)(struct sk_buff *msg, struct ib_mr *ibmr);
        int (*fill_res_cq_entry)(struct sk_buff *msg, struct ib_cq *ibcq);
        int (*fill_res_cq_entry_raw)(struct sk_buff *msg, struct ib_cq *ibcq);
        int (*fill_res_qp_entry)(struct sk_buff *msg, struct ib_qp *ibqp);
        int (*fill_res_qp_entry_raw)(struct sk_buff *msg, struct ib_qp *ibqp);
        int (*fill_res_cm_id_entry)(struct sk_buff *msg, struct rdma_cm_id *id);
        int (*fill_res_srq_entry)(struct sk_buff *msg, struct ib_srq *ib_srq);
        int (*fill_res_srq_entry_raw)(struct sk_buff *msg, struct ib_srq *ib_srq);

        /* Device lifecycle callbacks */
        /*
         * Called after the device becomes registered, before clients are
         * attached
         */
        int (*enable_driver)(struct ib_device *dev);
        /*
         * This is called as part of ib_dealloc_device().
         */
        void (*dealloc_driver)(struct ib_device *dev);

        /* iWarp CM callbacks */
        void (*iw_add_ref)(struct ib_qp *qp);
        void (*iw_rem_ref)(struct ib_qp *qp);
        struct ib_qp *(*iw_get_qp)(struct ib_device *device, int qpn);
        int (*iw_connect)(struct iw_cm_id *cm_id,
                          struct iw_cm_conn_param *conn_param);
        int (*iw_accept)(struct iw_cm_id *cm_id,
                         struct iw_cm_conn_param *conn_param);
        int (*iw_reject)(struct iw_cm_id *cm_id, const void *pdata,
                         u8 pdata_len);
        int (*iw_create_listen)(struct iw_cm_id *cm_id, int backlog);
        int (*iw_destroy_listen)(struct iw_cm_id *cm_id);
        /**
         * counter_bind_qp - Bind a QP to a counter.
         * @counter - The counter to be bound. If counter->id is zero then
         *   the driver needs to allocate a new counter and set counter->id
         */
        int (*counter_bind_qp)(struct rdma_counter *counter, struct ib_qp *qp,
                               u32 port);
        /**
         * counter_unbind_qp - Unbind the qp from the dynamically-allocated
         *   counter and bind it onto the default one
         */
        int (*counter_unbind_qp)(struct ib_qp *qp, u32 port);
        /**
         * counter_dealloc -De-allocate the hw counter
         */
        int (*counter_dealloc)(struct rdma_counter *counter);
        /**
         * counter_alloc_stats - Allocate a struct rdma_hw_stats and fill in
         * the driver initialized data.
         */
        struct rdma_hw_stats *(*counter_alloc_stats)(
                struct rdma_counter *counter);
        /**
         * counter_update_stats - Query the stats value of this counter
         */
        int (*counter_update_stats)(struct rdma_counter *counter);

        /**
         * counter_init - Initialize the driver specific rdma counter struct.
         */
        void (*counter_init)(struct rdma_counter *counter);

        /**
         * Allows rdma drivers to add their own restrack attributes
         * dumped via 'rdma stat' iproute2 command.
         */
        int (*fill_stat_mr_entry)(struct sk_buff *msg, struct ib_mr *ibmr);

        /* query driver for its ucontext properties */
        int (*query_ucontext)(struct ib_ucontext *context,
                              struct uverbs_attr_bundle *attrs);

        /*
         * Provide NUMA node. This API exists for rdmavt/hfi1 only.
         * Everyone else relies on Linux memory management model.
         */
        int (*get_numa_node)(struct ib_device *dev);

        /**
         * add_sub_dev - Add a sub IB device
         */
        struct ib_device *(*add_sub_dev)(struct ib_device *parent,
                                         enum rdma_nl_dev_type type,
                                         const char *name);

        /**
         * del_sub_dev - Delete a sub IB device
         */
        void (*del_sub_dev)(struct ib_device *sub_dev);

        /**
         * ufile_cleanup - Attempt to cleanup ubojects HW resources inside
         * the ufile.
         */
        void (*ufile_hw_cleanup)(struct ib_uverbs_file *ufile);

        /**
         * report_port_event - Drivers need to implement this if they have
         * some private stuff to handle when link status changes.
         */
        void (*report_port_event)(struct ib_device *ibdev,
                                  struct net_device *ndev, unsigned long event);

        DECLARE_RDMA_OBJ_SIZE(ib_ah);
        DECLARE_RDMA_OBJ_SIZE(ib_counters);
        DECLARE_RDMA_OBJ_SIZE(ib_cq);
        DECLARE_RDMA_OBJ_SIZE(ib_mw);
        DECLARE_RDMA_OBJ_SIZE(ib_pd);
        DECLARE_RDMA_OBJ_SIZE(ib_qp);
        DECLARE_RDMA_OBJ_SIZE(ib_rwq_ind_table);
        DECLARE_RDMA_OBJ_SIZE(ib_srq);
        DECLARE_RDMA_OBJ_SIZE(ib_ucontext);
        DECLARE_RDMA_OBJ_SIZE(ib_xrcd);
        DECLARE_RDMA_OBJ_SIZE(rdma_counter);
};

struct ib_core_device {
        /* device must be the first element in structure until,
         * union of ib_core_device and device exists in ib_device.
         */
        struct device dev;
        possible_net_t rdma_net;
        struct kobject *ports_kobj;
        struct list_head port_list;
        struct ib_device *owner; /* reach back to owner ib_device */
};

struct rdma_restrack_root;
struct ib_device {
        /* Do not access @dma_device directly from ULP nor from HW drivers. */
        struct device                *dma_device;
        struct ib_device_ops             ops;
        char                          name[IB_DEVICE_NAME_MAX];
        struct rcu_head rcu_head;

        struct list_head              event_handler_list;
        /* Protects event_handler_list */
        struct rw_semaphore event_handler_rwsem;

        /* Protects QP's event_handler calls and open_qp list */
        spinlock_t qp_open_list_lock;

        struct rw_semaphore              client_data_rwsem;
        struct xarray                 client_data;
        struct mutex                  unregistration_lock;

        /* Synchronize GID, Pkey cache entries, subnet prefix, LMC */
        rwlock_t cache_lock;
        /**
         * port_data is indexed by port number
         */
        struct ib_port_data *port_data;

        int                              num_comp_vectors;

        union {
                struct device                dev;
                struct ib_core_device        coredev;
        };

        /* First group is for device attributes,
         * Second group is for driver provided attributes (optional).
         * Third group is for the hw_stats
         * It is a NULL terminated array.
         */
        const struct attribute_group        *groups[4];
        u8                                hw_stats_attr_index;

        u64                             uverbs_cmd_mask;

        char                             node_desc[IB_DEVICE_NODE_DESC_MAX];
        __be64                             node_guid;
        u32                             local_dma_lkey;
        u16                          is_switch:1;
        /* Indicates kernel verbs support, should not be used in drivers */
        u16                          kverbs_provider:1;
        /* CQ adaptive moderation (RDMA DIM) */
        u16                          use_cq_dim:1;
        u8                           node_type;
        u32                             phys_port_cnt;
        struct ib_device_attr        attrs;
        struct hw_stats_device_data *hw_stats_data;

#ifdef CONFIG_CGROUP_RDMA
        struct rdmacg_device         cg_device;
#endif

        u32                          index;

        spinlock_t                   cq_pools_lock;
        struct list_head             cq_pools[IB_POLL_LAST_POOL_TYPE + 1];

        struct rdma_restrack_root *res;

        const struct uapi_definition   *driver_def;

        /*
         * Positive refcount indicates that the device is currently
         * registered and cannot be unregistered.
         */
        refcount_t refcount;
        struct completion unreg_completion;
        struct work_struct unregistration_work;

        const struct rdma_link_ops *link_ops;

        /* Protects compat_devs xarray modifications */
        struct mutex compat_devs_mutex;
        /* Maintains compat devices for each net namespace */
        struct xarray compat_devs;

        /* Used by iWarp CM */
        char iw_ifname[IFNAMSIZ];
        u32 iw_driver_flags;
        u32 lag_flags;

        /* A parent device has a list of sub-devices */
        struct mutex subdev_lock;
        struct list_head subdev_list_head;

        /* A sub device has a type and a parent */
        enum rdma_nl_dev_type type;
        struct ib_device *parent;
        struct list_head subdev_list;

        enum rdma_nl_name_assign_type name_assign_type;
};

static inline void *rdma_zalloc_obj(struct ib_device *dev, size_t size,
                                    gfp_t gfp, bool is_numa_aware)
{
        if (is_numa_aware && dev->ops.get_numa_node)
                return kzalloc_node(size, gfp, dev->ops.get_numa_node(dev));

        return kzalloc(size, gfp);
}

struct ib_client_nl_info;
struct ib_client {
        const char *name;
        int (*add)(struct ib_device *ibdev);
        void (*remove)(struct ib_device *, void *client_data);
        void (*rename)(struct ib_device *dev, void *client_data);
        int (*get_nl_info)(struct ib_device *ibdev, void *client_data,
                           struct ib_client_nl_info *res);
        int (*get_global_nl_info)(struct ib_client_nl_info *res);

        /* Returns the net_dev belonging to this ib_client and matching the
         * given parameters.
         * @dev:         An RDMA device that the net_dev use for communication.
         * @port:         A physical port number on the RDMA device.
         * @pkey:         P_Key that the net_dev uses if applicable.
         * @gid:         A GID that the net_dev uses to communicate.
         * @addr:         An IP address the net_dev is configured with.
         * @client_data: The device's client data set by ib_set_client_data().
         *
         * An ib_client that implements a net_dev on top of RDMA devices
         * (such as IP over IB) should implement this callback, allowing the
         * rdma_cm module to find the right net_dev for a given request.
         *
         * The caller is responsible for calling dev_put on the returned
         * netdev. */
        struct net_device *(*get_net_dev_by_params)(
                        struct ib_device *dev,
                        u32 port,
                        u16 pkey,
                        const union ib_gid *gid,
                        const struct sockaddr *addr,
                        void *client_data);

        refcount_t uses;
        struct completion uses_zero;
        u32 client_id;

        /* kverbs are not required by the client */
        u8 no_kverbs_req:1;
};

/*
 * IB block DMA iterator
 *
 * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
 * to a HW supported page size.
 */
struct ib_block_iter {
        /* internal states */
        struct scatterlist *__sg;        /* sg holding the current aligned block */
        dma_addr_t __dma_addr;                /* unaligned DMA address of this block */
        size_t __sg_numblocks;                /* ib_umem_num_dma_blocks() */
        unsigned int __sg_nents;        /* number of SG entries */
        unsigned int __sg_advance;        /* number of bytes to advance in sg in next step */
        unsigned int __pg_bit;                /* alignment of current block */
};

struct ib_device *_ib_alloc_device(size_t size);
#define ib_alloc_device(drv_struct, member)                                    \
        container_of(_ib_alloc_device(sizeof(struct drv_struct) +              \
                                      BUILD_BUG_ON_ZERO(offsetof(              \
                                              struct drv_struct, member))),    \
                     struct drv_struct, member)

void ib_dealloc_device(struct ib_device *device);

void ib_get_device_fw_str(struct ib_device *device, char *str);

int ib_register_device(struct ib_device *device, const char *name,
                       struct device *dma_device);
void ib_unregister_device(struct ib_device *device);
void ib_unregister_driver(enum rdma_driver_id driver_id);
void ib_unregister_device_and_put(struct ib_device *device);
void ib_unregister_device_queued(struct ib_device *ib_dev);

int ib_register_client   (struct ib_client *client);
void ib_unregister_client(struct ib_client *client);

void __rdma_block_iter_start(struct ib_block_iter *biter,
                             struct scatterlist *sglist,
                             unsigned int nents,
                             unsigned long pgsz);
bool __rdma_block_iter_next(struct ib_block_iter *biter);

/**
 * rdma_block_iter_dma_address - get the aligned dma address of the current
 * block held by the block iterator.
 * @biter: block iterator holding the memory block
 */
static inline dma_addr_t
rdma_block_iter_dma_address(struct ib_block_iter *biter)
{
        return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
}

/**
 * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
 * @sglist: sglist to iterate over
 * @biter: block iterator holding the memory block
 * @nents: maximum number of sg entries to iterate over
 * @pgsz: best HW supported page size to use
 *
 * Callers may use rdma_block_iter_dma_address() to get each
 * blocks aligned DMA address.
 */
#define rdma_for_each_block(sglist, biter, nents, pgsz)                \
        for (__rdma_block_iter_start(biter, sglist, nents,        \
                                     pgsz);                        \
             __rdma_block_iter_next(biter);)

/**
 * ib_get_client_data - Get IB client context
 * @device:Device to get context for
 * @client:Client to get context for
 *
 * ib_get_client_data() returns the client context data set with
 * ib_set_client_data(). This can only be called while the client is
 * registered to the device, once the ib_client remove() callback returns this
 * cannot be called.
 */
static inline void *ib_get_client_data(struct ib_device *device,
                                       struct ib_client *client)
{
        return xa_load(&device->client_data, client->client_id);
}
void  ib_set_client_data(struct ib_device *device, struct ib_client *client,
                         void *data);
void ib_set_device_ops(struct ib_device *device,
                       const struct ib_device_ops *ops);

int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
                      unsigned long pfn, unsigned long size, pgprot_t prot,
                      struct rdma_user_mmap_entry *entry);
int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
                                struct rdma_user_mmap_entry *entry,
                                size_t length);
int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
                                      struct rdma_user_mmap_entry *entry,
                                      size_t length, u32 min_pgoff,
                                      u32 max_pgoff);

#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
void rdma_user_mmap_disassociate(struct ib_device *device);
#else
static inline void rdma_user_mmap_disassociate(struct ib_device *device)
{
}
#endif

static inline int
rdma_user_mmap_entry_insert_exact(struct ib_ucontext *ucontext,
                                  struct rdma_user_mmap_entry *entry,
                                  size_t length, u32 pgoff)
{
        return rdma_user_mmap_entry_insert_range(ucontext, entry, length, pgoff,
                                                 pgoff);
}

struct rdma_user_mmap_entry *
rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
                               unsigned long pgoff);
struct rdma_user_mmap_entry *
rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
                         struct vm_area_struct *vma);
void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry);

void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry);

static inline int ib_copy_from_udata(void *dest, struct ib_udata *udata, size_t len)
{
        return copy_from_user(dest, udata->inbuf, len) ? -EFAULT : 0;
}

static inline int ib_copy_to_udata(struct ib_udata *udata, void *src, size_t len)
{
        return copy_to_user(udata->outbuf, src, len) ? -EFAULT : 0;
}

static inline bool ib_is_buffer_cleared(const void __user *p,
                                        size_t len)
{
        bool ret;
        u8 *buf;

        if (len > USHRT_MAX)
                return false;

        buf = memdup_user(p, len);
        if (IS_ERR(buf))
                return false;

        ret = !memchr_inv(buf, 0, len);
        kfree(buf);
        return ret;
}

static inline bool ib_is_udata_cleared(struct ib_udata *udata,
                                       size_t offset,
                                       size_t len)
{
        return ib_is_buffer_cleared(udata->inbuf + offset, len);
}

/**
 * ib_modify_qp_is_ok - Check that the supplied attribute mask
 * contains all required attributes and no attributes not allowed for
 * the given QP state transition.
 * @cur_state: Current QP state
 * @next_state: Next QP state
 * @type: QP type
 * @mask: Mask of supplied QP attributes
 *
 * This function is a helper function that a low-level driver's
 * modify_qp method can use to validate the consumer's input.  It
 * checks that cur_state and next_state are valid QP states, that a
 * transition from cur_state to next_state is allowed by the IB spec,
 * and that the attribute mask supplied is allowed for the transition.
 */
bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
                        enum ib_qp_type type, enum ib_qp_attr_mask mask);

void ib_register_event_handler(struct ib_event_handler *event_handler);
void ib_unregister_event_handler(struct ib_event_handler *event_handler);
void ib_dispatch_event(const struct ib_event *event);

int ib_query_port(struct ib_device *device,
                  u32 port_num, struct ib_port_attr *port_attr);

enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
                                               u32 port_num);

/**
 * rdma_cap_ib_switch - Check if the device is IB switch
 * @device: Device to check
 *
 * Device driver is responsible for setting is_switch bit on
 * in ib_device structure at init time.
 *
 * Return: true if the device is IB switch.
 */
static inline bool rdma_cap_ib_switch(const struct ib_device *device)
{
        return device->is_switch;
}

/**
 * rdma_start_port - Return the first valid port number for the device
 * specified
 *
 * @device: Device to be checked
 *
 * Return start port number
 */
static inline u32 rdma_start_port(const struct ib_device *device)
{
        return rdma_cap_ib_switch(device) ? 0 : 1;
}

/**
 * rdma_for_each_port - Iterate over all valid port numbers of the IB device
 * @device - The struct ib_device * to iterate over
 * @iter - The unsigned int to store the port number
 */
#define rdma_for_each_port(device, iter)                                       \
        for (iter = rdma_start_port(device +                                       \
                                    BUILD_BUG_ON_ZERO(!__same_type(u32,               \
                                                                   iter)));    \
             iter <= rdma_end_port(device); iter++)

/**
 * rdma_end_port - Return the last valid port number for the device
 * specified
 *
 * @device: Device to be checked
 *
 * Return last port number
 */
static inline u32 rdma_end_port(const struct ib_device *device)
{
        return rdma_cap_ib_switch(device) ? 0 : device->phys_port_cnt;
}

static inline int rdma_is_port_valid(const struct ib_device *device,
                                     unsigned int port)
{
        return (port >= rdma_start_port(device) &&
                port <= rdma_end_port(device));
}

static inline bool rdma_is_grh_required(const struct ib_device *device,
                                        u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_PORT_IB_GRH_REQUIRED;
}

static inline bool rdma_protocol_ib(const struct ib_device *device,
                                    u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_IB;
}

static inline bool rdma_protocol_roce(const struct ib_device *device,
                                      u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               (RDMA_CORE_CAP_PROT_ROCE | RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP);
}

static inline bool rdma_protocol_roce_udp_encap(const struct ib_device *device,
                                                u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
}

static inline bool rdma_protocol_roce_eth_encap(const struct ib_device *device,
                                                u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_ROCE;
}

static inline bool rdma_protocol_iwarp(const struct ib_device *device,
                                       u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_IWARP;
}

static inline bool rdma_ib_or_roce(const struct ib_device *device,
                                   u32 port_num)
{
        return rdma_protocol_ib(device, port_num) ||
                rdma_protocol_roce(device, port_num);
}

static inline bool rdma_protocol_raw_packet(const struct ib_device *device,
                                            u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_RAW_PACKET;
}

static inline bool rdma_protocol_usnic(const struct ib_device *device,
                                       u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_PROT_USNIC;
}

/**
 * rdma_cap_ib_mad - Check if the port of a device supports Infiniband
 * Management Datagrams.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * Management Datagrams (MAD) are a required part of the InfiniBand
 * specification and are supported on all InfiniBand devices.  A slightly
 * extended version are also supported on OPA interfaces.
 *
 * Return: true if the port supports sending/receiving of MAD packets.
 */
static inline bool rdma_cap_ib_mad(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_IB_MAD;
}

/**
 * rdma_cap_opa_mad - Check if the port of device provides support for OPA
 * Management Datagrams.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * Intel OmniPath devices extend and/or replace the InfiniBand Management
 * datagrams with their own versions.  These OPA MADs share many but not all of
 * the characteristics of InfiniBand MADs.
 *
 * OPA MADs differ in the following ways:
 *
 *    1) MADs are variable size up to 2K
 *       IBTA defined MADs remain fixed at 256 bytes
 *    2) OPA SMPs must carry valid PKeys
 *    3) OPA SMP packets are a different format
 *
 * Return: true if the port supports OPA MAD packet formats.
 */
static inline bool rdma_cap_opa_mad(struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
                RDMA_CORE_CAP_OPA_MAD;
}

/**
 * rdma_cap_ib_smi - Check if the port of a device provides an Infiniband
 * Subnet Management Agent (SMA) on the Subnet Management Interface (SMI).
 * @device: Device to check
 * @port_num: Port number to check
 *
 * Each InfiniBand node is required to provide a Subnet Management Agent
 * that the subnet manager can access.  Prior to the fabric being fully
 * configured by the subnet manager, the SMA is accessed via a well known
 * interface called the Subnet Management Interface (SMI).  This interface
 * uses directed route packets to communicate with the SM to get around the
 * chicken and egg problem of the SM needing to know what's on the fabric
 * in order to configure the fabric, and needing to configure the fabric in
 * order to send packets to the devices on the fabric.  These directed
 * route packets do not need the fabric fully configured in order to reach
 * their destination.  The SMI is the only method allowed to send
 * directed route packets on an InfiniBand fabric.
 *
 * Return: true if the port provides an SMI.
 */
static inline bool rdma_cap_ib_smi(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_IB_SMI;
}

/**
 * rdma_cap_ib_cm - Check if the port of device has the capability Infiniband
 * Communication Manager.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * The InfiniBand Communication Manager is one of many pre-defined General
 * Service Agents (GSA) that are accessed via the General Service
 * Interface (GSI).  It's role is to facilitate establishment of connections
 * between nodes as well as other management related tasks for established
 * connections.
 *
 * Return: true if the port supports an IB CM (this does not guarantee that
 * a CM is actually running however).
 */
static inline bool rdma_cap_ib_cm(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_IB_CM;
}

/**
 * rdma_cap_iw_cm - Check if the port of device has the capability IWARP
 * Communication Manager.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * Similar to above, but specific to iWARP connections which have a different
 * managment protocol than InfiniBand.
 *
 * Return: true if the port supports an iWARP CM (this does not guarantee that
 * a CM is actually running however).
 */
static inline bool rdma_cap_iw_cm(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_IW_CM;
}

/**
 * rdma_cap_ib_sa - Check if the port of device has the capability Infiniband
 * Subnet Administration.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * An InfiniBand Subnet Administration (SA) service is a pre-defined General
 * Service Agent (GSA) provided by the Subnet Manager (SM).  On InfiniBand
 * fabrics, devices should resolve routes to other hosts by contacting the
 * SA to query the proper route.
 *
 * Return: true if the port should act as a client to the fabric Subnet
 * Administration interface.  This does not imply that the SA service is
 * running locally.
 */
static inline bool rdma_cap_ib_sa(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_IB_SA;
}

/**
 * rdma_cap_ib_mcast - Check if the port of device has the capability Infiniband
 * Multicast.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * InfiniBand multicast registration is more complex than normal IPv4 or
 * IPv6 multicast registration.  Each Host Channel Adapter must register
 * with the Subnet Manager when it wishes to join a multicast group.  It
 * should do so only once regardless of how many queue pairs it subscribes
 * to this group.  And it should leave the group only after all queue pairs
 * attached to the group have been detached.
 *
 * Return: true if the port must undertake the additional adminstrative
 * overhead of registering/unregistering with the SM and tracking of the
 * total number of queue pairs attached to the multicast group.
 */
static inline bool rdma_cap_ib_mcast(const struct ib_device *device,
                                     u32 port_num)
{
        return rdma_cap_ib_sa(device, port_num);
}

/**
 * rdma_cap_af_ib - Check if the port of device has the capability
 * Native Infiniband Address.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * InfiniBand addressing uses a port's GUID + Subnet Prefix to make a default
 * GID.  RoCE uses a different mechanism, but still generates a GID via
 * a prescribed mechanism and port specific data.
 *
 * Return: true if the port uses a GID address to identify devices on the
 * network.
 */
static inline bool rdma_cap_af_ib(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_AF_IB;
}

/**
 * rdma_cap_eth_ah - Check if the port of device has the capability
 * Ethernet Address Handle.
 * @device: Device to check
 * @port_num: Port number to check
 *
 * RoCE is InfiniBand over Ethernet, and it uses a well defined technique
 * to fabricate GIDs over Ethernet/IP specific addresses native to the
 * port.  Normally, packet headers are generated by the sending host
 * adapter, but when sending connectionless datagrams, we must manually
 * inject the proper headers for the fabric we are communicating over.
 *
 * Return: true if we are running as a RoCE port and must force the
 * addition of a Global Route Header built from our Ethernet Address
 * Handle into our header list for connectionless packets.
 */
static inline bool rdma_cap_eth_ah(const struct ib_device *device, u32 port_num)
{
        return device->port_data[port_num].immutable.core_cap_flags &
               RDMA_CORE_CAP_ETH_AH;
}

/**
 * rdma_cap_opa_ah - Check if the port of device supports
 * OPA Address handles
 * @device: Device to check
 * @port_num: Port number to check
 *
 * Return: true if we are running on an OPA device which supports
 * the extended OPA addressing.
 */
static inline bool rdma_cap_opa_ah(struct ib_device *device, u32 port_num)
{
        return (device->port_data[port_num].immutable.core_cap_flags &
                RDMA_CORE_CAP_OPA_AH) == RDMA_CORE_CAP_OPA_AH;
}

/**
 * rdma_max_mad_size - Return the max MAD size required by this RDMA Port.
 *
 * @device: Device
 * @port_num: Port number
 *
 * This MAD size includes the MAD headers and MAD payload.  No other headers
 * are included.
 *
 * Return the max MAD size required by the Port.  Will return 0 if the port
 * does not support MADs
 */
static inline size_t rdma_max_mad_size(const struct ib_device *device,
                                       u32 port_num)
{
        return device->port_data[port_num].immutable.max_mad_size;
}

/**
 * rdma_cap_roce_gid_table - Check if the port of device uses roce_gid_table
 * @device: Device to check
 * @port_num: Port number to check
 *
 * RoCE GID table mechanism manages the various GIDs for a device.
 *
 * NOTE: if allocating the port's GID table has failed, this call will still
 * return true, but any RoCE GID table API will fail.
 *
 * Return: true if the port uses RoCE GID table mechanism in order to manage
 * its GIDs.
 */
static inline bool rdma_cap_roce_gid_table(const struct ib_device *device,
                                           u32 port_num)
{
        return rdma_protocol_roce(device, port_num) &&
                device->ops.add_gid && device->ops.del_gid;
}

/*
 * Check if the device supports READ W/ INVALIDATE.
 */
static inline bool rdma_cap_read_inv(struct ib_device *dev, u32 port_num)
{
        /*
         * iWarp drivers must support READ W/ INVALIDATE.  No other protocol
         * has support for it yet.
         */
        return rdma_protocol_iwarp(dev, port_num);
}

/**
 * rdma_core_cap_opa_port - Return whether the RDMA Port is OPA or not.
 * @device: Device
 * @port_num: 1 based Port number
 *
 * Return true if port is an Intel OPA port , false if not
 */
static inline bool rdma_core_cap_opa_port(struct ib_device *device,
                                          u32 port_num)
{
        return (device->port_data[port_num].immutable.core_cap_flags &
                RDMA_CORE_PORT_INTEL_OPA) == RDMA_CORE_PORT_INTEL_OPA;
}

/**
 * rdma_mtu_enum_to_int - Return the mtu of the port as an integer value.
 * @device: Device
 * @port_num: Port number
 * @mtu: enum value of MTU
 *
 * Return the MTU size supported by the port as an integer value. Will return
 * -1 if enum value of mtu is not supported.
 */
static inline int rdma_mtu_enum_to_int(struct ib_device *device, u32 port,
                                       int mtu)
{
        if (rdma_core_cap_opa_port(device, port))
                return opa_mtu_enum_to_int((enum opa_mtu)mtu);
        else
                return ib_mtu_enum_to_int((enum ib_mtu)mtu);
}

/**
 * rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
 * @device: Device
 * @port_num: Port number
 * @attr: port attribute
 *
 * Return the MTU size supported by the port as an integer value.
 */
static inline int rdma_mtu_from_attr(struct ib_device *device, u32 port,
                                     struct ib_port_attr *attr)
{
        if (rdma_core_cap_opa_port(device, port))
                return attr->phys_mtu;
        else
                return ib_mtu_enum_to_int(attr->max_mtu);
}

int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
                         int state);
int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
                     struct ifla_vf_info *info);
int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
                    struct ifla_vf_stats *stats);
int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
                    struct ifla_vf_guid *node_guid,
                    struct ifla_vf_guid *port_guid);
int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
                   int type);

int ib_query_pkey(struct ib_device *device,
                  u32 port_num, u16 index, u16 *pkey);

int ib_modify_device(struct ib_device *device,
                     int device_modify_mask,
                     struct ib_device_modify *device_modify);

int ib_modify_port(struct ib_device *device,
                   u32 port_num, int port_modify_mask,
                   struct ib_port_modify *port_modify);

int ib_find_gid(struct ib_device *device, union ib_gid *gid,
                u32 *port_num, u16 *index);

int ib_find_pkey(struct ib_device *device,
                 u32 port_num, u16 pkey, u16 *index);

enum ib_pd_flags {
        /*
         * Create a memory registration for all memory in the system and place
         * the rkey for it into pd->unsafe_global_rkey.  This can be used by
         * ULPs to avoid the overhead of dynamic MRs.
         *
         * This flag is generally considered unsafe and must only be used in
         * extremly trusted environments.  Every use of it will log a warning
         * in the kernel log.
         */
        IB_PD_UNSAFE_GLOBAL_RKEY        = 0x01,
};

struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
                const char *caller);

/**
 * ib_alloc_pd - Allocates an unused protection domain.
 * @device: The device on which to allocate the protection domain.
 * @flags: protection domain flags
 *
 * A protection domain object provides an association between QPs, shared
 * receive queues, address handles, memory regions, and memory windows.
 *
 * Every PD has a local_dma_lkey which can be used as the lkey value for local
 * memory operations.
 */
#define ib_alloc_pd(device, flags) \
        __ib_alloc_pd((device), (flags), KBUILD_MODNAME)

int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata);

/**
 * ib_dealloc_pd - Deallocate kernel PD
 * @pd: The protection domain
 *
 * NOTE: for user PD use ib_dealloc_pd_user with valid udata!
 */
static inline void ib_dealloc_pd(struct ib_pd *pd)
{
        int ret = ib_dealloc_pd_user(pd, NULL);

        WARN_ONCE(ret, "Destroy of kernel PD shouldn't fail");
}

enum rdma_create_ah_flags {
        /* In a sleepable context */
        RDMA_CREATE_AH_SLEEPABLE = BIT(0),
};

/**
 * rdma_create_ah - Creates an address handle for the given address vector.
 * @pd: The protection domain associated with the address handle.
 * @ah_attr: The attributes of the address vector.
 * @flags: Create address handle flags (see enum rdma_create_ah_flags).
 *
 * The address handle is used to reference a local or global destination
 * in all UD QP post sends.
 */
struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
                             u32 flags);

/**
 * rdma_create_user_ah - Creates an address handle for the given address vector.
 * It resolves destination mac address for ah attribute of RoCE type.
 * @pd: The protection domain associated with the address handle.
 * @ah_attr: The attributes of the address vector.
 * @udata: pointer to user's input output buffer information need by
 *         provider driver.
 *
 * It returns 0 on success and returns appropriate error code on error.
 * The address handle is used to reference a local or global destination
 * in all UD QP post sends.
 */
struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
                                  struct rdma_ah_attr *ah_attr,
                                  struct ib_udata *udata);
/**
 * ib_get_gids_from_rdma_hdr - Get sgid and dgid from GRH or IPv4 header
 *   work completion.
 * @hdr: the L3 header to parse
 * @net_type: type of header to parse
 * @sgid: place to store source gid
 * @dgid: place to store destination gid
 */
int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
                              enum rdma_network_type net_type,
                              union ib_gid *sgid, union ib_gid *dgid);

/**
 * ib_get_rdma_header_version - Get the header version
 * @hdr: the L3 header to parse
 */
int ib_get_rdma_header_version(const union rdma_network_hdr *hdr);

/**
 * ib_init_ah_attr_from_wc - Initializes address handle attributes from a
 *   work completion.
 * @device: Device on which the received message arrived.
 * @port_num: Port on which the received message arrived.
 * @wc: Work completion associated with the received message.
 * @grh: References the received global route header.  This parameter is
 *   ignored unless the work completion indicates that the GRH is valid.
 * @ah_attr: Returned attributes that can be used when creating an address
 *   handle for replying to the message.
 * When ib_init_ah_attr_from_wc() returns success,
 * (a) for IB link layer it optionally contains a reference to SGID attribute
 * when GRH is present for IB link layer.
 * (b) for RoCE link layer it contains a reference to SGID attribute.
 * User must invoke rdma_cleanup_ah_attr_gid_attr() to release reference to SGID
 * attributes which are initialized using ib_init_ah_attr_from_wc().
 *
 */
int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
                            const struct ib_wc *wc, const struct ib_grh *grh,
                            struct rdma_ah_attr *ah_attr);

/**
 * ib_create_ah_from_wc - Creates an address handle associated with the
 *   sender of the specified work completion.
 * @pd: The protection domain associated with the address handle.
 * @wc: Work completion information associated with a received message.
 * @grh: References the received global route header.  This parameter is
 *   ignored unless the work completion indicates that the GRH is valid.
 * @port_num: The outbound port number to associate with the address.
 *
 * The address handle is used to reference a local or global destination
 * in all UD QP post sends.
 */
struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
                                   const struct ib_grh *grh, u32 port_num);

/**
 * rdma_modify_ah - Modifies the address vector associated with an address
 *   handle.
 * @ah: The address handle to modify.
 * @ah_attr: The new address vector attributes to associate with the
 *   address handle.
 */
int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);

/**
 * rdma_query_ah - Queries the address vector associated with an address
 *   handle.
 * @ah: The address handle to query.
 * @ah_attr: The address vector attributes associated with the address
 *   handle.
 */
int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr);

enum rdma_destroy_ah_flags {
        /* In a sleepable context */
        RDMA_DESTROY_AH_SLEEPABLE = BIT(0),
};

/**
 * rdma_destroy_ah_user - Destroys an address handle.
 * @ah: The address handle to destroy.
 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
 * @udata: Valid user data or NULL for kernel objects
 */
int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata);

/**
 * rdma_destroy_ah - Destroys an kernel address handle.
 * @ah: The address handle to destroy.
 * @flags: Destroy address handle flags (see enum rdma_destroy_ah_flags).
 *
 * NOTE: for user ah use rdma_destroy_ah_user with valid udata!
 */
static inline void rdma_destroy_ah(struct ib_ah *ah, u32 flags)
{
        int ret = rdma_destroy_ah_user(ah, flags, NULL);

        WARN_ONCE(ret, "Destroy of kernel AH shouldn't fail");
}

struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
                                  struct ib_srq_init_attr *srq_init_attr,
                                  struct ib_usrq_object *uobject,
                                  struct ib_udata *udata);
static inline struct ib_srq *
ib_create_srq(struct ib_pd *pd, struct ib_srq_init_attr *srq_init_attr)
{
        if (!pd->device->ops.create_srq)
                return ERR_PTR(-EOPNOTSUPP);

        return ib_create_srq_user(pd, srq_init_attr, NULL, NULL);
}

/**
 * ib_modify_srq - Modifies the attributes for the specified SRQ.
 * @srq: The SRQ to modify.
 * @srq_attr: On input, specifies the SRQ attributes to modify.  On output,
 *   the current values of selected SRQ attributes are returned.
 * @srq_attr_mask: A bit-mask used to specify which attributes of the SRQ
 *   are being modified.
 *
 * The mask may contain IB_SRQ_MAX_WR to resize the SRQ and/or
 * IB_SRQ_LIMIT to set the SRQ's limit and request notification when
 * the number of receives queued drops below the limit.
 */
int ib_modify_srq(struct ib_srq *srq,
                  struct ib_srq_attr *srq_attr,
                  enum ib_srq_attr_mask srq_attr_mask);

/**
 * ib_query_srq - Returns the attribute list and current values for the
 *   specified SRQ.
 * @srq: The SRQ to query.
 * @srq_attr: The attributes of the specified SRQ.
 */
int ib_query_srq(struct ib_srq *srq,
                 struct ib_srq_attr *srq_attr);

/**
 * ib_destroy_srq_user - Destroys the specified SRQ.
 * @srq: The SRQ to destroy.
 * @udata: Valid user data or NULL for kernel objects
 */
int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata);

/**
 * ib_destroy_srq - Destroys the specified kernel SRQ.
 * @srq: The SRQ to destroy.
 *
 * NOTE: for user srq use ib_destroy_srq_user with valid udata!
 */
static inline void ib_destroy_srq(struct ib_srq *srq)
{
        int ret = ib_destroy_srq_user(srq, NULL);

        WARN_ONCE(ret, "Destroy of kernel SRQ shouldn't fail");
}

/**
 * ib_post_srq_recv - Posts a list of work requests to the specified SRQ.
 * @srq: The SRQ to post the work request on.
 * @recv_wr: A list of work requests to post on the receive queue.
 * @bad_recv_wr: On an immediate failure, this parameter will reference
 *   the work request that failed to be posted on the QP.
 */
static inline int ib_post_srq_recv(struct ib_srq *srq,
                                   const struct ib_recv_wr *recv_wr,
                                   const struct ib_recv_wr **bad_recv_wr)
{
        const struct ib_recv_wr *dummy;

        return srq->device->ops.post_srq_recv(srq, recv_wr,
                                              bad_recv_wr ? : &dummy);
}

struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
                                  struct ib_qp_init_attr *qp_init_attr,
                                  const char *caller);
/**
 * ib_create_qp - Creates a kernel QP associated with the specific protection
 * domain.
 * @pd: The protection domain associated with the QP.
 * @init_attr: A list of initial attributes required to create the
 *   QP.  If QP creation succeeds, then the attributes are updated to
 *   the actual capabilities of the created QP.
 */
static inline struct ib_qp *ib_create_qp(struct ib_pd *pd,
                                         struct ib_qp_init_attr *init_attr)
{
        return ib_create_qp_kernel(pd, init_attr, KBUILD_MODNAME);
}

/**
 * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
 * @qp: The QP to modify.
 * @attr: On input, specifies the QP attributes to modify.  On output,
 *   the current values of selected QP attributes are returned.
 * @attr_mask: A bit-mask used to specify which attributes of the QP
 *   are being modified.
 * @udata: pointer to user's input output buffer information
 *   are being modified.
 * It returns 0 on success and returns appropriate error code on error.
 */
int ib_modify_qp_with_udata(struct ib_qp *qp,
                            struct ib_qp_attr *attr,
                            int attr_mask,
                            struct ib_udata *udata);

/**
 * ib_modify_qp - Modifies the attributes for the specified QP and then
 *   transitions the QP to the given state.
 * @qp: The QP to modify.
 * @qp_attr: On input, specifies the QP attributes to modify.  On output,
 *   the current values of selected QP attributes are returned.
 * @qp_attr_mask: A bit-mask used to specify which attributes of the QP
 *   are being modified.
 */
int ib_modify_qp(struct ib_qp *qp,
                 struct ib_qp_attr *qp_attr,
                 int qp_attr_mask);

/**
 * ib_query_qp - Returns the attribute list and current values for the
 *   specified QP.
 * @qp: The QP to query.
 * @qp_attr: The attributes of the specified QP.
 * @qp_attr_mask: A bit-mask used to select specific attributes to query.
 * @qp_init_attr: Additional attributes of the selected QP.
 *
 * The qp_attr_mask may be used to limit the query to gathering only the
 * selected attributes.
 */
int ib_query_qp(struct ib_qp *qp,
                struct ib_qp_attr *qp_attr,
                int qp_attr_mask,
                struct ib_qp_init_attr *qp_init_attr);

/**
 * ib_destroy_qp - Destroys the specified QP.
 * @qp: The QP to destroy.
 * @udata: Valid udata or NULL for kernel objects
 */
int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata);

/**
 * ib_destroy_qp - Destroys the specified kernel QP.
 * @qp: The QP to destroy.
 *
 * NOTE: for user qp use ib_destroy_qp_user with valid udata!
 */
static inline int ib_destroy_qp(struct ib_qp *qp)
{
        return ib_destroy_qp_user(qp, NULL);
}

/**
 * ib_open_qp - Obtain a reference to an existing sharable QP.
 * @xrcd - XRC domain
 * @qp_open_attr: Attributes identifying the QP to open.
 *
 * Returns a reference to a sharable QP.
 */
struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
                         struct ib_qp_open_attr *qp_open_attr);

/**
 * ib_close_qp - Release an external reference to a QP.
 * @qp: The QP handle to release
 *
 * The opened QP handle is released by the caller.  The underlying
 * shared QP is not destroyed until all internal references are released.
 */
int ib_close_qp(struct ib_qp *qp);

/**
 * ib_post_send - Posts a list of work requests to the send queue of
 *   the specified QP.
 * @qp: The QP to post the work request on.
 * @send_wr: A list of work requests to post on the send queue.
 * @bad_send_wr: On an immediate failure, this parameter will reference
 *   the work request that failed to be posted on the QP.
 *
 * While IBA Vol. 1 section 11.4.1.1 specifies that if an immediate
 * error is returned, the QP state shall not be affected,
 * ib_post_send() will return an immediate error after queueing any
 * earlier work requests in the list.
 */
static inline int ib_post_send(struct ib_qp *qp,
                               const struct ib_send_wr *send_wr,
                               const struct ib_send_wr **bad_send_wr)
{
        const struct ib_send_wr *dummy;

        return qp->device->ops.post_send(qp, send_wr, bad_send_wr ? : &dummy);
}

/**
 * ib_post_recv - Posts a list of work requests to the receive queue of
 *   the specified QP.
 * @qp: The QP to post the work request on.
 * @recv_wr: A list of work requests to post on the receive queue.
 * @bad_recv_wr: On an immediate failure, this parameter will reference
 *   the work request that failed to be posted on the QP.
 */
static inline int ib_post_recv(struct ib_qp *qp,
                               const struct ib_recv_wr *recv_wr,
                               const struct ib_recv_wr **bad_recv_wr)
{
        const struct ib_recv_wr *dummy;

        return qp->device->ops.post_recv(qp, recv_wr, bad_recv_wr ? : &dummy);
}

struct ib_cq *__ib_alloc_cq(struct ib_device *dev, void *private, int nr_cqe,
                            int comp_vector, enum ib_poll_context poll_ctx,
                            const char *caller);
static inline struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private,
                                        int nr_cqe, int comp_vector,
                                        enum ib_poll_context poll_ctx)
{
        return __ib_alloc_cq(dev, private, nr_cqe, comp_vector, poll_ctx,
                             KBUILD_MODNAME);
}

struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
                                int nr_cqe, enum ib_poll_context poll_ctx,
                                const char *caller);

/**
 * ib_alloc_cq_any: Allocate kernel CQ
 * @dev: The IB device
 * @private: Private data attached to the CQE
 * @nr_cqe: Number of CQEs in the CQ
 * @poll_ctx: Context used for polling the CQ
 */
static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
                                            void *private, int nr_cqe,
                                            enum ib_poll_context poll_ctx)
{
        return __ib_alloc_cq_any(dev, private, nr_cqe, poll_ctx,
                                 KBUILD_MODNAME);
}

void ib_free_cq(struct ib_cq *cq);
int ib_process_cq_direct(struct ib_cq *cq, int budget);

/**
 * ib_create_cq - Creates a CQ on the specified device.
 * @device: The device on which to create the CQ.
 * @comp_handler: A user-specified callback that is invoked when a
 *   completion event occurs on the CQ.
 * @event_handler: A user-specified callback that is invoked when an
 *   asynchronous event not associated with a completion occurs on the CQ.
 * @cq_context: Context associated with the CQ returned to the user via
 *   the associated completion and event handlers.
 * @cq_attr: The attributes the CQ should be created upon.
 *
 * Users can examine the cq structure to determine the actual CQ size.
 */
struct ib_cq *__ib_create_cq(struct ib_device *device,
                             ib_comp_handler comp_handler,
                             void (*event_handler)(struct ib_event *, void *),
                             void *cq_context,
                             const struct ib_cq_init_attr *cq_attr,
                             const char *caller);
#define ib_create_cq(device, cmp_hndlr, evt_hndlr, cq_ctxt, cq_attr) \
        __ib_create_cq((device), (cmp_hndlr), (evt_hndlr), (cq_ctxt), (cq_attr), KBUILD_MODNAME)

/**
 * ib_resize_cq - Modifies the capacity of the CQ.
 * @cq: The CQ to resize.
 * @cqe: The minimum size of the CQ.
 *
 * Users can examine the cq structure to determine the actual CQ size.
 */
int ib_resize_cq(struct ib_cq *cq, int cqe);

/**
 * rdma_set_cq_moderation - Modifies moderation params of the CQ
 * @cq: The CQ to modify.
 * @cq_count: number of CQEs that will trigger an event
 * @cq_period: max period of time in usec before triggering an event
 *
 */
int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);

/**
 * ib_destroy_cq_user - Destroys the specified CQ.
 * @cq: The CQ to destroy.
 * @udata: Valid user data or NULL for kernel objects
 */
int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);

/**
 * ib_destroy_cq - Destroys the specified kernel CQ.
 * @cq: The CQ to destroy.
 *
 * NOTE: for user cq use ib_destroy_cq_user with valid udata!
 */
static inline void ib_destroy_cq(struct ib_cq *cq)
{
        int ret = ib_destroy_cq_user(cq, NULL);

        WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
}

/**
 * ib_poll_cq - poll a CQ for completion(s)
 * @cq:the CQ being polled
 * @num_entries:maximum number of completions to return
 * @wc:array of at least @num_entries &struct ib_wc where completions
 *   will be returned
 *
 * Poll a CQ for (possibly multiple) completions.  If the return value
 * is < 0, an error occurred.  If the return value is >= 0, it is the
 * number of completions returned.  If the return value is
 * non-negative and < num_entries, then the CQ was emptied.
 */
static inline int ib_poll_cq(struct ib_cq *cq, int num_entries,
                             struct ib_wc *wc)
{
        return cq->device->ops.poll_cq(cq, num_entries, wc);
}

/**
 * ib_req_notify_cq - Request completion notification on a CQ.
 * @cq: The CQ to generate an event for.
 * @flags:
 *   Must contain exactly one of %IB_CQ_SOLICITED or %IB_CQ_NEXT_COMP
 *   to request an event on the next solicited event or next work
 *   completion at any type, respectively. %IB_CQ_REPORT_MISSED_EVENTS
 *   may also be |ed in to request a hint about missed events, as
 *   described below.
 *
 * Return Value:
 *    < 0 means an error occurred while requesting notification
 *   == 0 means notification was requested successfully, and if
 *        IB_CQ_REPORT_MISSED_EVENTS was passed in, then no events
 *        were missed and it is safe to wait for another event.  In
 *        this case is it guaranteed that any work completions added
 *        to the CQ since the last CQ poll will trigger a completion
 *        notification event.
 *    > 0 is only returned if IB_CQ_REPORT_MISSED_EVENTS was passed
 *        in.  It means that the consumer must poll the CQ again to
 *        make sure it is empty to avoid missing an event because of a
 *        race between requesting notification and an entry being
 *        added to the CQ.  This return value means it is possible
 *        (but not guaranteed) that a work completion has been added
 *        to the CQ since the last poll without triggering a
 *        completion notification event.
 */
static inline int ib_req_notify_cq(struct ib_cq *cq,
                                   enum ib_cq_notify_flags flags)
{
        return cq->device->ops.req_notify_cq(cq, flags);
}

struct ib_cq *ib_cq_pool_get(struct ib_device *dev, unsigned int nr_cqe,
                             int comp_vector_hint,
                             enum ib_poll_context poll_ctx);

void ib_cq_pool_put(struct ib_cq *cq, unsigned int nr_cqe);

/*
 * Drivers that don't need a DMA mapping at the RDMA layer, set dma_device to
 * NULL. This causes the ib_dma* helpers to just stash the kernel virtual
 * address into the dma address.
 */
static inline bool ib_uses_virt_dma(struct ib_device *dev)
{
        return IS_ENABLED(CONFIG_INFINIBAND_VIRT_DMA) && !dev->dma_device;
}

/*
 * Check if a IB device's underlying DMA mapping supports P2PDMA transfers.
 */
static inline bool ib_dma_pci_p2p_dma_supported(struct ib_device *dev)
{
        if (ib_uses_virt_dma(dev))
                return false;

        return dma_pci_p2pdma_supported(dev->dma_device);
}

/**
 * ib_virt_dma_to_ptr - Convert a dma_addr to a kernel pointer
 * @dma_addr: The DMA address
 *
 * Used by ib_uses_virt_dma() devices to get back to the kernel pointer after
 * going through the dma_addr marshalling.
 */
static inline void *ib_virt_dma_to_ptr(u64 dma_addr)
{
        /* virt_dma mode maps the kvs's directly into the dma addr */
        return (void *)(uintptr_t)dma_addr;
}

/**
 * ib_virt_dma_to_page - Convert a dma_addr to a struct page
 * @dma_addr: The DMA address
 *
 * Used by ib_uses_virt_dma() device to get back to the struct page after going
 * through the dma_addr marshalling.
 */
static inline struct page *ib_virt_dma_to_page(u64 dma_addr)
{
        return virt_to_page(ib_virt_dma_to_ptr(dma_addr));
}

/**
 * ib_dma_mapping_error - check a DMA addr for error
 * @dev: The device for which the dma_addr was created
 * @dma_addr: The DMA address to check
 */
static inline int ib_dma_mapping_error(struct ib_device *dev, u64 dma_addr)
{
        if (ib_uses_virt_dma(dev))
                return 0;
        return dma_mapping_error(dev->dma_device, dma_addr);
}

/**
 * ib_dma_map_single - Map a kernel virtual address to DMA address
 * @dev: The device for which the dma_addr is to be created
 * @cpu_addr: The kernel virtual address
 * @size: The size of the region in bytes
 * @direction: The direction of the DMA
 */
static inline u64 ib_dma_map_single(struct ib_device *dev,
                                    void *cpu_addr, size_t size,
                                    enum dma_data_direction direction)
{
        if (ib_uses_virt_dma(dev))
                return (uintptr_t)cpu_addr;
        return dma_map_single(dev->dma_device, cpu_addr, size, direction);
}

/**
 * ib_dma_unmap_single - Destroy a mapping created by ib_dma_map_single()
 * @dev: The device for which the DMA address was created
 * @addr: The DMA address
 * @size: The size of the region in bytes
 * @direction: The direction of the DMA
 */
static inline void ib_dma_unmap_single(struct ib_device *dev,
                                       u64 addr, size_t size,
                                       enum dma_data_direction direction)
{
        if (!ib_uses_virt_dma(dev))
                dma_unmap_single(dev->dma_device, addr, size, direction);
}

/**
 * ib_dma_map_page - Map a physical page to DMA address
 * @dev: The device for which the dma_addr is to be created
 * @page: The page to be mapped
 * @offset: The offset within the page
 * @size: The size of the region in bytes
 * @direction: The direction of the DMA
 */
static inline u64 ib_dma_map_page(struct ib_device *dev,
                                  struct page *page,
                                  unsigned long offset,
                                  size_t size,
                                         enum dma_data_direction direction)
{
        if (ib_uses_virt_dma(dev))
                return (uintptr_t)(page_address(page) + offset);
        return dma_map_page(dev->dma_device, page, offset, size, direction);
}

/**
 * ib_dma_unmap_page - Destroy a mapping created by ib_dma_map_page()
 * @dev: The device for which the DMA address was created
 * @addr: The DMA address
 * @size: The size of the region in bytes
 * @direction: The direction of the DMA
 */
static inline void ib_dma_unmap_page(struct ib_device *dev,
                                     u64 addr, size_t size,
                                     enum dma_data_direction direction)
{
        if (!ib_uses_virt_dma(dev))
                dma_unmap_page(dev->dma_device, addr, size, direction);
}

int ib_dma_virt_map_sg(struct ib_device *dev, struct scatterlist *sg, int nents);
static inline int ib_dma_map_sg_attrs(struct ib_device *dev,
                                      struct scatterlist *sg, int nents,
                                      enum dma_data_direction direction,
                                      unsigned long dma_attrs)
{
        if (ib_uses_virt_dma(dev))
                return ib_dma_virt_map_sg(dev, sg, nents);
        return dma_map_sg_attrs(dev->dma_device, sg, nents, direction,
                                dma_attrs);
}

static inline void ib_dma_unmap_sg_attrs(struct ib_device *dev,
                                         struct scatterlist *sg, int nents,
                                         enum dma_data_direction direction,
                                         unsigned long dma_attrs)
{
        if (!ib_uses_virt_dma(dev))
                dma_unmap_sg_attrs(dev->dma_device, sg, nents, direction,
                                   dma_attrs);
}

/**
 * ib_dma_map_sgtable_attrs - Map a scatter/gather table to DMA addresses
 * @dev: The device for which the DMA addresses are to be created
 * @sg: The sg_table object describing the buffer
 * @direction: The direction of the DMA
 * @attrs: Optional DMA attributes for the map operation
 */
static inline int ib_dma_map_sgtable_attrs(struct ib_device *dev,
                                           struct sg_table *sgt,
                                           enum dma_data_direction direction,
                                           unsigned long dma_attrs)
{
        int nents;

        if (ib_uses_virt_dma(dev)) {
                nents = ib_dma_virt_map_sg(dev, sgt->sgl, sgt->orig_nents);
                if (!nents)
                        return -EIO;
                sgt->nents = nents;
                return 0;
        }
        return dma_map_sgtable(dev->dma_device, sgt, direction, dma_attrs);
}

static inline void ib_dma_unmap_sgtable_attrs(struct ib_device *dev,
                                              struct sg_table *sgt,
                                              enum dma_data_direction direction,
                                              unsigned long dma_attrs)
{
        if (!ib_uses_virt_dma(dev))
                dma_unmap_sgtable(dev->dma_device, sgt, direction, dma_attrs);
}

/**
 * ib_dma_map_sg - Map a scatter/gather list to DMA addresses
 * @dev: The device for which the DMA addresses are to be created
 * @sg: The array of scatter/gather entries
 * @nents: The number of scatter/gather entries
 * @direction: The direction of the DMA
 */
static inline int ib_dma_map_sg(struct ib_device *dev,
                                struct scatterlist *sg, int nents,
                                enum dma_data_direction direction)
{
        return ib_dma_map_sg_attrs(dev, sg, nents, direction, 0);
}

/**
 * ib_dma_unmap_sg - Unmap a scatter/gather list of DMA addresses
 * @dev: The device for which the DMA addresses were created
 * @sg: The array of scatter/gather entries
 * @nents: The number of scatter/gather entries
 * @direction: The direction of the DMA
 */
static inline void ib_dma_unmap_sg(struct ib_device *dev,
                                   struct scatterlist *sg, int nents,
                                   enum dma_data_direction direction)
{
        ib_dma_unmap_sg_attrs(dev, sg, nents, direction, 0);
}

/**
 * ib_dma_max_seg_size - Return the size limit of a single DMA transfer
 * @dev: The device to query
 *
 * The returned value represents a size in bytes.
 */
static inline unsigned int ib_dma_max_seg_size(struct ib_device *dev)
{
        if (ib_uses_virt_dma(dev))
                return UINT_MAX;
        return dma_get_max_seg_size(dev->dma_device);
}

/**
 * ib_dma_sync_single_for_cpu - Prepare DMA region to be accessed by CPU
 * @dev: The device for which the DMA address was created
 * @addr: The DMA address
 * @size: The size of the region in bytes
 * @dir: The direction of the DMA
 */
static inline void ib_dma_sync_single_for_cpu(struct ib_device *dev,
                                              u64 addr,
                                              size_t size,
                                              enum dma_data_direction dir)
{
        if (!ib_uses_virt_dma(dev))
                dma_sync_single_for_cpu(dev->dma_device, addr, size, dir);
}

/**
 * ib_dma_sync_single_for_device - Prepare DMA region to be accessed by device
 * @dev: The device for which the DMA address was created
 * @addr: The DMA address
 * @size: The size of the region in bytes
 * @dir: The direction of the DMA
 */
static inline void ib_dma_sync_single_for_device(struct ib_device *dev,
                                                 u64 addr,
                                                 size_t size,
                                                 enum dma_data_direction dir)
{
        if (!ib_uses_virt_dma(dev))
                dma_sync_single_for_device(dev->dma_device, addr, size, dir);
}

/* ib_reg_user_mr - register a memory region for virtual addresses from kernel
 * space. This function should be called when 'current' is the owning MM.
 */
struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
                             u64 virt_addr, int mr_access_flags);

/* ib_advise_mr -  give an advice about an address range in a memory region */
int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
                 u32 flags, struct ib_sge *sg_list, u32 num_sge);
/**
 * ib_dereg_mr_user - Deregisters a memory region and removes it from the
 *   HCA translation table.
 * @mr: The memory region to deregister.
 * @udata: Valid user data or NULL for kernel object
 *
 * This function can fail, if the memory region has memory windows bound to it.
 */
int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata);

/**
 * ib_dereg_mr - Deregisters a kernel memory region and removes it from the
 *   HCA translation table.
 * @mr: The memory region to deregister.
 *
 * This function can fail, if the memory region has memory windows bound to it.
 *
 * NOTE: for user mr use ib_dereg_mr_user with valid udata!
 */
static inline int ib_dereg_mr(struct ib_mr *mr)
{
        return ib_dereg_mr_user(mr, NULL);
}

struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
                          u32 max_num_sg);

struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
                                    u32 max_num_data_sg,
                                    u32 max_num_meta_sg);

/**
 * ib_update_fast_reg_key - updates the key portion of the fast_reg MR
 *   R_Key and L_Key.
 * @mr - struct ib_mr pointer to be updated.
 * @newkey - new key to be used.
 */
static inline void ib_update_fast_reg_key(struct ib_mr *mr, u8 newkey)
{
        mr->lkey = (mr->lkey & 0xffffff00) | newkey;
        mr->rkey = (mr->rkey & 0xffffff00) | newkey;
}

/**
 * ib_inc_rkey - increments the key portion of the given rkey. Can be used
 * for calculating a new rkey for type 2 memory windows.
 * @rkey - the rkey to increment.
 */
static inline u32 ib_inc_rkey(u32 rkey)
{
        const u32 mask = 0x000000ff;
        return ((rkey + 1) & mask) | (rkey & ~mask);
}

/**
 * ib_attach_mcast - Attaches the specified QP to a multicast group.
 * @qp: QP to attach to the multicast group.  The QP must be type
 *   IB_QPT_UD.
 * @gid: Multicast group GID.
 * @lid: Multicast group LID in host byte order.
 *
 * In order to send and receive multicast packets, subnet
 * administration must have created the multicast group and configured
 * the fabric appropriately.  The port associated with the specified
 * QP must also be a member of the multicast group.
 */
int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);

/**
 * ib_detach_mcast - Detaches the specified QP from a multicast group.
 * @qp: QP to detach from the multicast group.
 * @gid: Multicast group GID.
 * @lid: Multicast group LID in host byte order.
 */
int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);

struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
                                   struct inode *inode, struct ib_udata *udata);
int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata);

static inline int ib_check_mr_access(struct ib_device *ib_dev,
                                     unsigned int flags)
{
        u64 device_cap = ib_dev->attrs.device_cap_flags;

        /*
         * Local write permission is required if remote write or
         * remote atomic permission is also requested.
         */
        if (flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) &&
            !(flags & IB_ACCESS_LOCAL_WRITE))
                return -EINVAL;

        if (flags & ~IB_ACCESS_SUPPORTED)
                return -EINVAL;

        if (flags & IB_ACCESS_ON_DEMAND &&
            !(ib_dev->attrs.kernel_cap_flags & IBK_ON_DEMAND_PAGING))
                return -EOPNOTSUPP;

        if ((flags & IB_ACCESS_FLUSH_GLOBAL &&
            !(device_cap & IB_DEVICE_FLUSH_GLOBAL)) ||
            (flags & IB_ACCESS_FLUSH_PERSISTENT &&
            !(device_cap & IB_DEVICE_FLUSH_PERSISTENT)))
                return -EOPNOTSUPP;

        return 0;
}

static inline bool ib_access_writable(int access_flags)
{
        /*
         * We have writable memory backing the MR if any of the following
         * access flags are set.  "Local write" and "remote write" obviously
         * require write access.  "Remote atomic" can do things like fetch and
         * add, which will modify memory, and "MW bind" can change permissions
         * by binding a window.
         */
        return access_flags &
                (IB_ACCESS_LOCAL_WRITE   | IB_ACCESS_REMOTE_WRITE |
                 IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND);
}

/**
 * ib_check_mr_status: lightweight check of MR status.
 *     This routine may provide status checks on a selected
 *     ib_mr. first use is for signature status check.
 *
 * @mr: A memory region.
 * @check_mask: Bitmask of which checks to perform from
 *     ib_mr_status_check enumeration.
 * @mr_status: The container of relevant status checks.
 *     failed checks will be indicated in the status bitmask
 *     and the relevant info shall be in the error item.
 */
int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
                       struct ib_mr_status *mr_status);

/**
 * ib_device_try_get: Hold a registration lock
 * device: The device to lock
 *
 * A device under an active registration lock cannot become unregistered. It
 * is only possible to obtain a registration lock on a device that is fully
 * registered, otherwise this function returns false.
 *
 * The registration lock is only necessary for actions which require the
 * device to still be registered. Uses that only require the device pointer to
 * be valid should use get_device(&ibdev->dev) to hold the memory.
 *
 */
static inline bool ib_device_try_get(struct ib_device *dev)
{
        return refcount_inc_not_zero(&dev->refcount);
}

void ib_device_put(struct ib_device *device);
struct ib_device *ib_device_get_by_netdev(struct net_device *ndev,
                                          enum rdma_driver_id driver_id);
struct ib_device *ib_device_get_by_name(const char *name,
                                        enum rdma_driver_id driver_id);
struct net_device *ib_get_net_dev_by_params(struct ib_device *dev, u32 port,
                                            u16 pkey, const union ib_gid *gid,
                                            const struct sockaddr *addr);
int ib_device_set_netdev(struct ib_device *ib_dev, struct net_device *ndev,
                         unsigned int port);
struct net_device *ib_device_get_netdev(struct ib_device *ib_dev,
                                        u32 port);
int ib_query_netdev_port(struct ib_device *ibdev, struct net_device *ndev,
                         u32 *port);

static inline enum ib_port_state ib_get_curr_port_state(struct net_device *net_dev)
{
        return (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
                IB_PORT_ACTIVE : IB_PORT_DOWN;
}

void ib_dispatch_port_state_event(struct ib_device *ibdev,
                                  struct net_device *ndev);
struct ib_wq *ib_create_wq(struct ib_pd *pd,
                           struct ib_wq_init_attr *init_attr);
int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata);

int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
                 unsigned int *sg_offset, unsigned int page_size);
int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
                    int data_sg_nents, unsigned int *data_sg_offset,
                    struct scatterlist *meta_sg, int meta_sg_nents,
                    unsigned int *meta_sg_offset, unsigned int page_size);

static inline int
ib_map_mr_sg_zbva(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
                  unsigned int *sg_offset, unsigned int page_size)
{
        int n;

        n = ib_map_mr_sg(mr, sg, sg_nents, sg_offset, page_size);
        mr->iova = 0;

        return n;
}

int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
                unsigned int *sg_offset, int (*set_page)(struct ib_mr *, u64));

void ib_drain_rq(struct ib_qp *qp);
void ib_drain_sq(struct ib_qp *qp);
void ib_drain_qp(struct ib_qp *qp);

int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed,
                     u8 *width);

static inline u8 *rdma_ah_retrieve_dmac(struct rdma_ah_attr *attr)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_ROCE)
                return attr->roce.dmac;
        return NULL;
}

static inline void rdma_ah_set_dlid(struct rdma_ah_attr *attr, u32 dlid)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_IB)
                attr->ib.dlid = (u16)dlid;
        else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                attr->opa.dlid = dlid;
}

static inline u32 rdma_ah_get_dlid(const struct rdma_ah_attr *attr)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_IB)
                return attr->ib.dlid;
        else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                return attr->opa.dlid;
        return 0;
}

static inline void rdma_ah_set_sl(struct rdma_ah_attr *attr, u8 sl)
{
        attr->sl = sl;
}

static inline u8 rdma_ah_get_sl(const struct rdma_ah_attr *attr)
{
        return attr->sl;
}

static inline void rdma_ah_set_path_bits(struct rdma_ah_attr *attr,
                                         u8 src_path_bits)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_IB)
                attr->ib.src_path_bits = src_path_bits;
        else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                attr->opa.src_path_bits = src_path_bits;
}

static inline u8 rdma_ah_get_path_bits(const struct rdma_ah_attr *attr)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_IB)
                return attr->ib.src_path_bits;
        else if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                return attr->opa.src_path_bits;
        return 0;
}

static inline void rdma_ah_set_make_grd(struct rdma_ah_attr *attr,
                                        bool make_grd)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                attr->opa.make_grd = make_grd;
}

static inline bool rdma_ah_get_make_grd(const struct rdma_ah_attr *attr)
{
        if (attr->type == RDMA_AH_ATTR_TYPE_OPA)
                return attr->opa.make_grd;
        return false;
}

static inline void rdma_ah_set_port_num(struct rdma_ah_attr *attr, u32 port_num)
{
        attr->port_num = port_num;
}

static inline u32 rdma_ah_get_port_num(const struct rdma_ah_attr *attr)
{
        return attr->port_num;
}

static inline void rdma_ah_set_static_rate(struct rdma_ah_attr *attr,
                                           u8 static_rate)
{
        attr->static_rate = static_rate;
}

static inline u8 rdma_ah_get_static_rate(const struct rdma_ah_attr *attr)
{
        return attr->static_rate;
}

static inline void rdma_ah_set_ah_flags(struct rdma_ah_attr *attr,
                                        enum ib_ah_flags flag)
{
        attr->ah_flags = flag;
}

static inline enum ib_ah_flags
                rdma_ah_get_ah_flags(const struct rdma_ah_attr *attr)
{
        return attr->ah_flags;
}

static inline const struct ib_global_route
                *rdma_ah_read_grh(const struct rdma_ah_attr *attr)
{
        return &attr->grh;
}

/*To retrieve and modify the grh */
static inline struct ib_global_route
                *rdma_ah_retrieve_grh(struct rdma_ah_attr *attr)
{
        return &attr->grh;
}

static inline void rdma_ah_set_dgid_raw(struct rdma_ah_attr *attr, void *dgid)
{
        struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

        memcpy(grh->dgid.raw, dgid, sizeof(grh->dgid));
}

static inline void rdma_ah_set_subnet_prefix(struct rdma_ah_attr *attr,
                                             __be64 prefix)
{
        struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

        grh->dgid.global.subnet_prefix = prefix;
}

static inline void rdma_ah_set_interface_id(struct rdma_ah_attr *attr,
                                            __be64 if_id)
{
        struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

        grh->dgid.global.interface_id = if_id;
}

static inline void rdma_ah_set_grh(struct rdma_ah_attr *attr,
                                   union ib_gid *dgid, u32 flow_label,
                                   u8 sgid_index, u8 hop_limit,
                                   u8 traffic_class)
{
        struct ib_global_route *grh = rdma_ah_retrieve_grh(attr);

        attr->ah_flags = IB_AH_GRH;
        if (dgid)
                grh->dgid = *dgid;
        grh->flow_label = flow_label;
        grh->sgid_index = sgid_index;
        grh->hop_limit = hop_limit;
        grh->traffic_class = traffic_class;
        grh->sgid_attr = NULL;
}

void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr);
void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
                             u32 flow_label, u8 hop_limit, u8 traffic_class,
                             const struct ib_gid_attr *sgid_attr);
void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
                       const struct rdma_ah_attr *src);
void rdma_replace_ah_attr(struct rdma_ah_attr *old,
                          const struct rdma_ah_attr *new);
void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src);

/**
 * rdma_ah_find_type - Return address handle type.
 *
 * @dev: Device to be checked
 * @port_num: Port number
 */
static inline enum rdma_ah_attr_type rdma_ah_find_type(struct ib_device *dev,
                                                       u32 port_num)
{
        if (rdma_protocol_roce(dev, port_num))
                return RDMA_AH_ATTR_TYPE_ROCE;
        if (rdma_protocol_ib(dev, port_num)) {
                if (rdma_cap_opa_ah(dev, port_num))
                        return RDMA_AH_ATTR_TYPE_OPA;
                return RDMA_AH_ATTR_TYPE_IB;
        }
        if (dev->type == RDMA_DEVICE_TYPE_SMI)
                return RDMA_AH_ATTR_TYPE_IB;

        return RDMA_AH_ATTR_TYPE_UNDEFINED;
}

/**
 * ib_lid_cpu16 - Return lid in 16bit CPU encoding.
 *     In the current implementation the only way to
 *     get the 32bit lid is from other sources for OPA.
 *     For IB, lids will always be 16bits so cast the
 *     value accordingly.
 *
 * @lid: A 32bit LID
 */
static inline u16 ib_lid_cpu16(u32 lid)
{
        WARN_ON_ONCE(lid & 0xFFFF0000);
        return (u16)lid;
}

/**
 * ib_lid_be16 - Return lid in 16bit BE encoding.
 *
 * @lid: A 32bit LID
 */
static inline __be16 ib_lid_be16(u32 lid)
{
        WARN_ON_ONCE(lid & 0xFFFF0000);
        return cpu_to_be16((u16)lid);
}

/**
 * ib_get_vector_affinity - Get the affinity mappings of a given completion
 *   vector
 * @device:         the rdma device
 * @comp_vector:    index of completion vector
 *
 * Returns NULL on failure, otherwise a corresponding cpu map of the
 * completion vector (returns all-cpus map if the device driver doesn't
 * implement get_vector_affinity).
 */
static inline const struct cpumask *
ib_get_vector_affinity(struct ib_device *device, int comp_vector)
{
        if (comp_vector < 0 || comp_vector >= device->num_comp_vectors ||
            !device->ops.get_vector_affinity)
                return NULL;

        return device->ops.get_vector_affinity(device, comp_vector);

}

/**
 * rdma_roce_rescan_device - Rescan all of the network devices in the system
 * and add their gids, as needed, to the relevant RoCE devices.
 *
 * @device:         the rdma device
 */
void rdma_roce_rescan_device(struct ib_device *ibdev);
void rdma_roce_rescan_port(struct ib_device *ib_dev, u32 port);
void roce_del_all_netdev_gids(struct ib_device *ib_dev,
                              u32 port, struct net_device *ndev);

struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile);

#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs);
#else
static inline int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
{
        return 0;
}
#endif

struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
                                     enum rdma_netdev_t type, const char *name,
                                     unsigned char name_assign_type,
                                     void (*setup)(struct net_device *));

int rdma_init_netdev(struct ib_device *device, u32 port_num,
                     enum rdma_netdev_t type, const char *name,
                     unsigned char name_assign_type,
                     void (*setup)(struct net_device *),
                     struct net_device *netdev);

/**
 * rdma_device_to_ibdev - Get ib_device pointer from device pointer
 *
 * @device:        device pointer for which ib_device pointer to retrieve
 *
 * rdma_device_to_ibdev() retrieves ib_device pointer from device.
 *
 */
static inline struct ib_device *rdma_device_to_ibdev(struct device *device)
{
        struct ib_core_device *coredev =
                container_of(device, struct ib_core_device, dev);

        return coredev->owner;
}

/**
 * ibdev_to_node - return the NUMA node for a given ib_device
 * @dev:        device to get the NUMA node for.
 */
static inline int ibdev_to_node(struct ib_device *ibdev)
{
        struct device *parent = ibdev->dev.parent;

        if (!parent)
                return NUMA_NO_NODE;
        return dev_to_node(parent);
}

/**
 * rdma_device_to_drv_device - Helper macro to reach back to driver's
 *                               ib_device holder structure from device pointer.
 *
 * NOTE: New drivers should not make use of this API; This API is only for
 * existing drivers who have exposed sysfs entries using
 * ops->device_group.
 */
#define rdma_device_to_drv_device(dev, drv_dev_struct, ibdev_member)           \
        container_of(rdma_device_to_ibdev(dev), drv_dev_struct, ibdev_member)

bool rdma_dev_access_netns(const struct ib_device *device,
                           const struct net *net);

#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)

/**
 * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
 *                               on the flow_label
 *
 * This function will convert the 20 bit flow_label input to a valid RoCE v2
 * UDP src port 14 bit value. All RoCE V2 drivers should use this same
 * convention.
 */
static inline u16 rdma_flow_label_to_udp_sport(u32 fl)
{
        u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;

        fl_low ^= fl_high >> 14;
        return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
}

/**
 * rdma_calc_flow_label - generate a RDMA symmetric flow label value based on
 *                        local and remote qpn values
 *
 * This function folded the multiplication results of two qpns, 24 bit each,
 * fields, and converts it to a 20 bit results.
 *
 * This function will create symmetric flow_label value based on the local
 * and remote qpn values. this will allow both the requester and responder
 * to calculate the same flow_label for a given connection.
 *
 * This helper function should be used by driver in case the upper layer
 * provide a zero flow_label value. This is to improve entropy of RDMA
 * traffic in the network.
 */
static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)
{
        u64 v = (u64)lqpn * rqpn;

        v ^= v >> 20;
        v ^= v >> 40;

        return (u32)(v & IB_GRH_FLOWLABEL_MASK);
}

/**
 * rdma_get_udp_sport - Calculate and set UDP source port based on the flow
 *                      label. If flow label is not defined in GRH then
 *                      calculate it based on lqpn/rqpn.
 *
 * @fl:                 flow label from GRH
 * @lqpn:               local qp number
 * @rqpn:               remote qp number
 */
static inline u16 rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
{
        if (!fl)
                fl = rdma_calc_flow_label(lqpn, rqpn);

        return rdma_flow_label_to_udp_sport(fl);
}

const struct ib_port_immutable*
ib_port_immutable_read(struct ib_device *dev, unsigned int port);

/** ib_add_sub_device - Add a sub IB device on an existing one
 *
 * @parent: The IB device that needs to add a sub device
 * @type: The type of the new sub device
 * @name: The name of the new sub device
 *
 *
 * Return 0 on success, an error code otherwise
 */
int ib_add_sub_device(struct ib_device *parent,
                      enum rdma_nl_dev_type type,
                      const char *name);


/** ib_del_sub_device_and_put - Delect an IB sub device while holding a 'get'
 *
 * @sub: The sub device that is going to be deleted
 *
 * Return 0 on success, an error code otherwise
 */
int ib_del_sub_device_and_put(struct ib_device *sub);

static inline void ib_mark_name_assigned_by_user(struct ib_device *ibdev)
{
        ibdev->name_assign_type = RDMA_NAME_ASSIGN_TYPE_USER;
}

#endif /* IB_VERBS_H */




































    1 


  258 


  258 

































  349 





  258 




  267 









  251 








  257 






















  351 
















  324 
    1 












  351 
















   32 
  231 




  351 










  351 


  350 


  351 





  266 

  266 


  259 
   24 


  351 







  348 







  351 


  330 


  351 



















  349 












  350 








  351 



  349 
  350 











    2 










    2 












    2 

    2 



































































































































































































































































  269 



  269 




































  269 















  265 




   85 








  137 

  137 










  259 


















  259 











  259 




















































    1 



  259 


















  258 











  259 












  257 





  257 









  259 


  259 
































   20 

   20 











  265 





















  229 





  259 




  259 






  259 



  257 









    1 






















  259 
  259 




  259 

  259 








  232 


  231 


















  259 






















  251 
  251 















  258 




  232 

  259 








  259 





  259 
















  257 































   63 







   62 







   21 

   21 

   20 
   20 







   20 

   20 



   20 
   20 





   63 










   63 













   96 




   84 


















   13 



   13 













   12 






















   84 




































































   12 
















   12 

   12 












































































































    1 











    1 























































    1 
















  269 




  269 





























   23 


   23 




    3 
    3 





   24 







   24 

   24 




























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2.
 * No bombay mix was harmed in the writing of this file.
 *
 * Copyright (C) 2020 Google LLC
 * Author: Will Deacon <will@kernel.org>
 */

#include <linux/bitfield.h>
#include <asm/kvm_pgtable.h>
#include <asm/stage2_pgtable.h>

struct kvm_pgtable_walk_data {
        struct kvm_pgtable_walker        *walker;

        const u64                        start;
        u64                                addr;
        const u64                        end;
};

static bool kvm_pgtable_walk_skip_bbm_tlbi(const struct kvm_pgtable_visit_ctx *ctx)
{
        return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_BBM_TLBI);
}

static bool kvm_pgtable_walk_skip_cmo(const struct kvm_pgtable_visit_ctx *ctx)
{
        return unlikely(ctx->flags & KVM_PGTABLE_WALK_SKIP_CMO);
}

static bool kvm_block_mapping_supported(const struct kvm_pgtable_visit_ctx *ctx, u64 phys)
{
        u64 granule = kvm_granule_size(ctx->level);

        if (!kvm_level_supports_block_mapping(ctx->level))
                return false;

        if (granule > (ctx->end - ctx->addr))
                return false;

        if (!IS_ALIGNED(phys, granule))
                return false;

        return IS_ALIGNED(ctx->addr, granule);
}

static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, s8 level)
{
        u64 shift = kvm_granule_shift(level);
        u64 mask = BIT(PAGE_SHIFT - 3) - 1;

        return (data->addr >> shift) & mask;
}

static u32 kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr)
{
        u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */
        u64 mask = BIT(pgt->ia_bits) - 1;

        return (addr & mask) >> shift;
}

static u32 kvm_pgd_pages(u32 ia_bits, s8 start_level)
{
        struct kvm_pgtable pgt = {
                .ia_bits        = ia_bits,
                .start_level        = start_level,
        };

        return kvm_pgd_page_idx(&pgt, -1ULL) + 1;
}

static bool kvm_pte_table(kvm_pte_t pte, s8 level)
{
        if (level == KVM_PGTABLE_LAST_LEVEL)
                return false;

        if (!kvm_pte_valid(pte))
                return false;

        return FIELD_GET(KVM_PTE_TYPE, pte) == KVM_PTE_TYPE_TABLE;
}

static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
{
        return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
}

static void kvm_clear_pte(kvm_pte_t *ptep)
{
        WRITE_ONCE(*ptep, 0);
}

static kvm_pte_t kvm_init_table_pte(kvm_pte_t *childp, struct kvm_pgtable_mm_ops *mm_ops)
{
        kvm_pte_t pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));

        pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);
        pte |= KVM_PTE_VALID;
        return pte;
}

static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, s8 level)
{
        kvm_pte_t pte = kvm_phys_to_pte(pa);
        u64 type = (level == KVM_PGTABLE_LAST_LEVEL) ? KVM_PTE_TYPE_PAGE :
                                                       KVM_PTE_TYPE_BLOCK;

        pte |= attr & (KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI);
        pte |= FIELD_PREP(KVM_PTE_TYPE, type);
        pte |= KVM_PTE_VALID;

        return pte;
}

static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)
{
        return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
}

static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data,
                                  const struct kvm_pgtable_visit_ctx *ctx,
                                  enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable_walker *walker = data->walker;

        /* Ensure the appropriate lock is held (e.g. RCU lock for stage-2 MMU) */
        WARN_ON_ONCE(kvm_pgtable_walk_shared(ctx) && !kvm_pgtable_walk_lock_held());
        return walker->cb(ctx, visit);
}

static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker,
                                      int r)
{
        /*
         * Visitor callbacks return EAGAIN when the conditions that led to a
         * fault are no longer reflected in the page tables due to a race to
         * update a PTE. In the context of a fault handler this is interpreted
         * as a signal to retry guest execution.
         *
         * Ignore the return code altogether for walkers outside a fault handler
         * (e.g. write protecting a range of memory) and chug along with the
         * page table walk.
         */
        if (r == -EAGAIN)
                return !(walker->flags & KVM_PGTABLE_WALK_HANDLE_FAULT);

        return !r;
}

static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
                              struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level);

static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
                                      struct kvm_pgtable_mm_ops *mm_ops,
                                      kvm_pteref_t pteref, s8 level)
{
        enum kvm_pgtable_walk_flags flags = data->walker->flags;
        kvm_pte_t *ptep = kvm_dereference_pteref(data->walker, pteref);
        struct kvm_pgtable_visit_ctx ctx = {
                .ptep        = ptep,
                .old        = READ_ONCE(*ptep),
                .arg        = data->walker->arg,
                .mm_ops        = mm_ops,
                .start        = data->start,
                .addr        = data->addr,
                .end        = data->end,
                .level        = level,
                .flags        = flags,
        };
        int ret = 0;
        bool reload = false;
        kvm_pteref_t childp;
        bool table = kvm_pte_table(ctx.old, level);

        if (table && (ctx.flags & KVM_PGTABLE_WALK_TABLE_PRE)) {
                ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_PRE);
                reload = true;
        }

        if (!table && (ctx.flags & KVM_PGTABLE_WALK_LEAF)) {
                ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_LEAF);
                reload = true;
        }

        /*
         * Reload the page table after invoking the walker callback for leaf
         * entries or after pre-order traversal, to allow the walker to descend
         * into a newly installed or replaced table.
         */
        if (reload) {
                ctx.old = READ_ONCE(*ptep);
                table = kvm_pte_table(ctx.old, level);
        }

        if (!kvm_pgtable_walk_continue(data->walker, ret))
                goto out;

        if (!table) {
                data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level));
                data->addr += kvm_granule_size(level);
                goto out;
        }

        childp = (kvm_pteref_t)kvm_pte_follow(ctx.old, mm_ops);
        ret = __kvm_pgtable_walk(data, mm_ops, childp, level + 1);
        if (!kvm_pgtable_walk_continue(data->walker, ret))
                goto out;

        if (ctx.flags & KVM_PGTABLE_WALK_TABLE_POST)
                ret = kvm_pgtable_visitor_cb(data, &ctx, KVM_PGTABLE_WALK_TABLE_POST);

out:
        if (kvm_pgtable_walk_continue(data->walker, ret))
                return 0;

        return ret;
}

static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
                              struct kvm_pgtable_mm_ops *mm_ops, kvm_pteref_t pgtable, s8 level)
{
        u32 idx;
        int ret = 0;

        if (WARN_ON_ONCE(level < KVM_PGTABLE_FIRST_LEVEL ||
                         level > KVM_PGTABLE_LAST_LEVEL))
                return -EINVAL;

        for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) {
                kvm_pteref_t pteref = &pgtable[idx];

                if (data->addr >= data->end)
                        break;

                ret = __kvm_pgtable_visit(data, mm_ops, pteref, level);
                if (ret)
                        break;
        }

        return ret;
}

static int _kvm_pgtable_walk(struct kvm_pgtable *pgt, struct kvm_pgtable_walk_data *data)
{
        u32 idx;
        int ret = 0;
        u64 limit = BIT(pgt->ia_bits);

        if (data->addr > limit || data->end > limit)
                return -ERANGE;

        if (!pgt->pgd)
                return -EINVAL;

        for (idx = kvm_pgd_page_idx(pgt, data->addr); data->addr < data->end; ++idx) {
                kvm_pteref_t pteref = &pgt->pgd[idx * PTRS_PER_PTE];

                ret = __kvm_pgtable_walk(data, pgt->mm_ops, pteref, pgt->start_level);
                if (ret)
                        break;
        }

        return ret;
}

int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
                     struct kvm_pgtable_walker *walker)
{
        struct kvm_pgtable_walk_data walk_data = {
                .start        = ALIGN_DOWN(addr, PAGE_SIZE),
                .addr        = ALIGN_DOWN(addr, PAGE_SIZE),
                .end        = PAGE_ALIGN(walk_data.addr + size),
                .walker        = walker,
        };
        int r;

        r = kvm_pgtable_walk_begin(walker);
        if (r)
                return r;

        r = _kvm_pgtable_walk(pgt, &walk_data);
        kvm_pgtable_walk_end(walker);

        return r;
}

struct leaf_walk_data {
        kvm_pte_t        pte;
        s8                level;
};

static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
                       enum kvm_pgtable_walk_flags visit)
{
        struct leaf_walk_data *data = ctx->arg;

        data->pte   = ctx->old;
        data->level = ctx->level;

        return 0;
}

int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
                         kvm_pte_t *ptep, s8 *level)
{
        struct leaf_walk_data data;
        struct kvm_pgtable_walker walker = {
                .cb        = leaf_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF,
                .arg        = &data,
        };
        int ret;

        ret = kvm_pgtable_walk(pgt, ALIGN_DOWN(addr, PAGE_SIZE),
                               PAGE_SIZE, &walker);
        if (!ret) {
                if (ptep)
                        *ptep  = data.pte;
                if (level)
                        *level = data.level;
        }

        return ret;
}

struct hyp_map_data {
        const u64                        phys;
        kvm_pte_t                        attr;
};

static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
{
        bool device = prot & KVM_PGTABLE_PROT_DEVICE;
        u32 mtype = device ? MT_DEVICE_nGnRE : MT_NORMAL;
        kvm_pte_t attr = FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_ATTRIDX, mtype);
        u32 sh = KVM_PTE_LEAF_ATTR_LO_S1_SH_IS;
        u32 ap = (prot & KVM_PGTABLE_PROT_W) ? KVM_PTE_LEAF_ATTR_LO_S1_AP_RW :
                                               KVM_PTE_LEAF_ATTR_LO_S1_AP_RO;

        if (!(prot & KVM_PGTABLE_PROT_R))
                return -EINVAL;

        if (prot & KVM_PGTABLE_PROT_X) {
                if (prot & KVM_PGTABLE_PROT_W)
                        return -EINVAL;

                if (device)
                        return -EINVAL;

                if (system_supports_bti_kernel())
                        attr |= KVM_PTE_LEAF_ATTR_HI_S1_GP;
        } else {
                attr |= KVM_PTE_LEAF_ATTR_HI_S1_XN;
        }

        attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap);
        if (!kvm_lpa2_is_enabled())
                attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh);
        attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF;
        attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW;
        *ptep = attr;

        return 0;
}

enum kvm_pgtable_prot kvm_pgtable_hyp_pte_prot(kvm_pte_t pte)
{
        enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW;
        u32 ap;

        if (!kvm_pte_valid(pte))
                return prot;

        if (!(pte & KVM_PTE_LEAF_ATTR_HI_S1_XN))
                prot |= KVM_PGTABLE_PROT_X;

        ap = FIELD_GET(KVM_PTE_LEAF_ATTR_LO_S1_AP, pte);
        if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RO)
                prot |= KVM_PGTABLE_PROT_R;
        else if (ap == KVM_PTE_LEAF_ATTR_LO_S1_AP_RW)
                prot |= KVM_PGTABLE_PROT_RW;

        return prot;
}

static bool hyp_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
                                    struct hyp_map_data *data)
{
        u64 phys = data->phys + (ctx->addr - ctx->start);
        kvm_pte_t new;

        if (!kvm_block_mapping_supported(ctx, phys))
                return false;

        new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
        if (ctx->old == new)
                return true;
        if (!kvm_pte_valid(ctx->old))
                ctx->mm_ops->get_page(ctx->ptep);
        else if (WARN_ON((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW))
                return false;

        smp_store_release(ctx->ptep, new);
        return true;
}

static int hyp_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
                          enum kvm_pgtable_walk_flags visit)
{
        kvm_pte_t *childp, new;
        struct hyp_map_data *data = ctx->arg;
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (hyp_map_walker_try_leaf(ctx, data))
                return 0;

        if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL))
                return -EINVAL;

        childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
        if (!childp)
                return -ENOMEM;

        new = kvm_init_table_pte(childp, mm_ops);
        mm_ops->get_page(ctx->ptep);
        smp_store_release(ctx->ptep, new);

        return 0;
}

int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
                        enum kvm_pgtable_prot prot)
{
        int ret;
        struct hyp_map_data map_data = {
                .phys        = ALIGN_DOWN(phys, PAGE_SIZE),
        };
        struct kvm_pgtable_walker walker = {
                .cb        = hyp_map_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF,
                .arg        = &map_data,
        };

        ret = hyp_set_prot_attr(prot, &map_data.attr);
        if (ret)
                return ret;

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        dsb(ishst);
        isb();
        return ret;
}

static int hyp_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
                            enum kvm_pgtable_walk_flags visit)
{
        kvm_pte_t *childp = NULL;
        u64 granule = kvm_granule_size(ctx->level);
        u64 *unmapped = ctx->arg;
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (!kvm_pte_valid(ctx->old))
                return -EINVAL;

        if (kvm_pte_table(ctx->old, ctx->level)) {
                childp = kvm_pte_follow(ctx->old, mm_ops);

                if (mm_ops->page_count(childp) != 1)
                        return 0;

                kvm_clear_pte(ctx->ptep);
                dsb(ishst);
                __tlbi_level(vae2is, __TLBI_VADDR(ctx->addr, 0), TLBI_TTL_UNKNOWN);
        } else {
                if (ctx->end - ctx->addr < granule)
                        return -EINVAL;

                kvm_clear_pte(ctx->ptep);
                dsb(ishst);
                __tlbi_level(vale2is, __TLBI_VADDR(ctx->addr, 0), ctx->level);
                *unmapped += granule;
        }

        dsb(ish);
        isb();
        mm_ops->put_page(ctx->ptep);

        if (childp)
                mm_ops->put_page(childp);

        return 0;
}

u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
        u64 unmapped = 0;
        struct kvm_pgtable_walker walker = {
                .cb        = hyp_unmap_walker,
                .arg        = &unmapped,
                .flags        = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
        };

        if (!pgt->mm_ops->page_count)
                return 0;

        kvm_pgtable_walk(pgt, addr, size, &walker);
        return unmapped;
}

int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
                         struct kvm_pgtable_mm_ops *mm_ops)
{
        s8 start_level = KVM_PGTABLE_LAST_LEVEL + 1 -
                         ARM64_HW_PGTABLE_LEVELS(va_bits);

        if (start_level < KVM_PGTABLE_FIRST_LEVEL ||
            start_level > KVM_PGTABLE_LAST_LEVEL)
                return -EINVAL;

        pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_page(NULL);
        if (!pgt->pgd)
                return -ENOMEM;

        pgt->ia_bits                = va_bits;
        pgt->start_level        = start_level;
        pgt->mm_ops                = mm_ops;
        pgt->mmu                = NULL;
        pgt->force_pte_cb        = NULL;

        return 0;
}

static int hyp_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
                           enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (!kvm_pte_valid(ctx->old))
                return 0;

        mm_ops->put_page(ctx->ptep);

        if (kvm_pte_table(ctx->old, ctx->level))
                mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));

        return 0;
}

void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
{
        struct kvm_pgtable_walker walker = {
                .cb        = hyp_free_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
        };

        WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
        pgt->mm_ops->put_page(kvm_dereference_pteref(&walker, pgt->pgd));
        pgt->pgd = NULL;
}

struct stage2_map_data {
        const u64                        phys;
        kvm_pte_t                        attr;
        u8                                owner_id;

        kvm_pte_t                        *anchor;
        kvm_pte_t                        *childp;

        struct kvm_s2_mmu                *mmu;
        void                                *memcache;

        /* Force mappings to page granularity */
        bool                                force_pte;

        /* Walk should update owner_id only */
        bool                                annotation;
};

u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift)
{
        u64 vtcr = VTCR_EL2_FLAGS;
        s8 lvls;

        vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT;
        vtcr |= VTCR_EL2_T0SZ(phys_shift);
        /*
         * Use a minimum 2 level page table to prevent splitting
         * host PMD huge pages at stage2.
         */
        lvls = stage2_pgtable_levels(phys_shift);
        if (lvls < 2)
                lvls = 2;

        /*
         * When LPA2 is enabled, the HW supports an extra level of translation
         * (for 5 in total) when using 4K pages. It also introduces VTCR_EL2.SL2
         * to as an addition to SL0 to enable encoding this extra start level.
         * However, since we always use concatenated pages for the first level
         * lookup, we will never need this extra level and therefore do not need
         * to touch SL2.
         */
        vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls);

#ifdef CONFIG_ARM64_HW_AFDBM
        /*
         * Enable the Hardware Access Flag management, unconditionally
         * on all CPUs. In systems that have asymmetric support for the feature
         * this allows KVM to leverage hardware support on the subset of cores
         * that implement the feature.
         *
         * The architecture requires VTCR_EL2.HA to be RES0 (thus ignored by
         * hardware) on implementations that do not advertise support for the
         * feature. As such, setting HA unconditionally is safe, unless you
         * happen to be running on a design that has unadvertised support for
         * HAFDBS. Here be dragons.
         */
        if (!cpus_have_final_cap(ARM64_WORKAROUND_AMPERE_AC03_CPU_38))
                vtcr |= VTCR_EL2_HA;
#endif /* CONFIG_ARM64_HW_AFDBM */

        if (kvm_lpa2_is_enabled())
                vtcr |= VTCR_EL2_DS;

        /* Set the vmid bits */
        vtcr |= (get_vmid_bits(mmfr1) == 16) ?
                VTCR_EL2_VS_16BIT :
                VTCR_EL2_VS_8BIT;

        return vtcr;
}

static bool stage2_has_fwb(struct kvm_pgtable *pgt)
{
        if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
                return false;

        return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);
}

void kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu,
                                phys_addr_t addr, size_t size)
{
        unsigned long pages, inval_pages;

        if (!system_supports_tlb_range()) {
                kvm_call_hyp(__kvm_tlb_flush_vmid, mmu);
                return;
        }

        pages = size >> PAGE_SHIFT;
        while (pages > 0) {
                inval_pages = min(pages, MAX_TLBI_RANGE_PAGES);
                kvm_call_hyp(__kvm_tlb_flush_vmid_range, mmu, addr, inval_pages);

                addr += inval_pages << PAGE_SHIFT;
                pages -= inval_pages;
        }
}

#define KVM_S2_MEMATTR(pgt, attr) PAGE_S2_MEMATTR(attr, stage2_has_fwb(pgt))

static int stage2_set_prot_attr(struct kvm_pgtable *pgt, enum kvm_pgtable_prot prot,
                                kvm_pte_t *ptep)
{
        kvm_pte_t attr;
        u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS;

        switch (prot & (KVM_PGTABLE_PROT_DEVICE |
                        KVM_PGTABLE_PROT_NORMAL_NC)) {
        case KVM_PGTABLE_PROT_DEVICE | KVM_PGTABLE_PROT_NORMAL_NC:
                return -EINVAL;
        case KVM_PGTABLE_PROT_DEVICE:
                if (prot & KVM_PGTABLE_PROT_X)
                        return -EINVAL;
                attr = KVM_S2_MEMATTR(pgt, DEVICE_nGnRE);
                break;
        case KVM_PGTABLE_PROT_NORMAL_NC:
                if (prot & KVM_PGTABLE_PROT_X)
                        return -EINVAL;
                attr = KVM_S2_MEMATTR(pgt, NORMAL_NC);
                break;
        default:
                attr = KVM_S2_MEMATTR(pgt, NORMAL);
        }

        if (!(prot & KVM_PGTABLE_PROT_X))
                attr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;

        if (prot & KVM_PGTABLE_PROT_R)
                attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;

        if (prot & KVM_PGTABLE_PROT_W)
                attr |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;

        if (!kvm_lpa2_is_enabled())
                attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh);

        attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
        attr |= prot & KVM_PTE_LEAF_ATTR_HI_SW;
        *ptep = attr;

        return 0;
}

enum kvm_pgtable_prot kvm_pgtable_stage2_pte_prot(kvm_pte_t pte)
{
        enum kvm_pgtable_prot prot = pte & KVM_PTE_LEAF_ATTR_HI_SW;

        if (!kvm_pte_valid(pte))
                return prot;

        if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R)
                prot |= KVM_PGTABLE_PROT_R;
        if (pte & KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W)
                prot |= KVM_PGTABLE_PROT_W;
        if (!(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN))
                prot |= KVM_PGTABLE_PROT_X;

        return prot;
}

static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
{
        if (!kvm_pte_valid(old) || !kvm_pte_valid(new))
                return true;

        return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS));
}

static bool stage2_pte_is_counted(kvm_pte_t pte)
{
        /*
         * The refcount tracks valid entries as well as invalid entries if they
         * encode ownership of a page to another entity than the page-table
         * owner, whose id is 0.
         */
        return !!pte;
}

static bool stage2_pte_is_locked(kvm_pte_t pte)
{
        return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED);
}

static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
{
        if (!kvm_pgtable_walk_shared(ctx)) {
                WRITE_ONCE(*ctx->ptep, new);
                return true;
        }

        return cmpxchg(ctx->ptep, ctx->old, new) == ctx->old;
}

/**
 * stage2_try_break_pte() - Invalidates a pte according to the
 *                            'break-before-make' requirements of the
 *                            architecture.
 *
 * @ctx: context of the visited pte.
 * @mmu: stage-2 mmu
 *
 * Returns: true if the pte was successfully broken.
 *
 * If the removed pte was valid, performs the necessary serialization and TLB
 * invalidation for the old value. For counted ptes, drops the reference count
 * on the containing table page.
 */
static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
                                 struct kvm_s2_mmu *mmu)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (stage2_pte_is_locked(ctx->old)) {
                /*
                 * Should never occur if this walker has exclusive access to the
                 * page tables.
                 */
                WARN_ON(!kvm_pgtable_walk_shared(ctx));
                return false;
        }

        if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED))
                return false;

        if (!kvm_pgtable_walk_skip_bbm_tlbi(ctx)) {
                /*
                 * Perform the appropriate TLB invalidation based on the
                 * evicted pte value (if any).
                 */
                if (kvm_pte_table(ctx->old, ctx->level)) {
                        u64 size = kvm_granule_size(ctx->level);
                        u64 addr = ALIGN_DOWN(ctx->addr, size);

                        kvm_tlb_flush_vmid_range(mmu, addr, size);
                } else if (kvm_pte_valid(ctx->old)) {
                        kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu,
                                     ctx->addr, ctx->level);
                }
        }

        if (stage2_pte_is_counted(ctx->old))
                mm_ops->put_page(ctx->ptep);

        return true;
}

static void stage2_make_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        WARN_ON(!stage2_pte_is_locked(*ctx->ptep));

        if (stage2_pte_is_counted(new))
                mm_ops->get_page(ctx->ptep);

        smp_store_release(ctx->ptep, new);
}

static bool stage2_unmap_defer_tlb_flush(struct kvm_pgtable *pgt)
{
        /*
         * If FEAT_TLBIRANGE is implemented, defer the individual
         * TLB invalidations until the entire walk is finished, and
         * then use the range-based TLBI instructions to do the
         * invalidations. Condition deferred TLB invalidation on the
         * system supporting FWB as the optimization is entirely
         * pointless when the unmap walker needs to perform CMOs.
         */
        return system_supports_tlb_range() && stage2_has_fwb(pgt);
}

static void stage2_unmap_put_pte(const struct kvm_pgtable_visit_ctx *ctx,
                                struct kvm_s2_mmu *mmu,
                                struct kvm_pgtable_mm_ops *mm_ops)
{
        struct kvm_pgtable *pgt = ctx->arg;

        /*
         * Clear the existing PTE, and perform break-before-make if it was
         * valid. Depending on the system support, defer the TLB maintenance
         * for the same until the entire unmap walk is completed.
         */
        if (kvm_pte_valid(ctx->old)) {
                kvm_clear_pte(ctx->ptep);

                if (kvm_pte_table(ctx->old, ctx->level)) {
                        kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
                                     TLBI_TTL_UNKNOWN);
                } else if (!stage2_unmap_defer_tlb_flush(pgt)) {
                        kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, ctx->addr,
                                     ctx->level);
                }
        }

        mm_ops->put_page(ctx->ptep);
}

static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte)
{
        u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR;
        return kvm_pte_valid(pte) && memattr == KVM_S2_MEMATTR(pgt, NORMAL);
}

static bool stage2_pte_executable(kvm_pte_t pte)
{
        return kvm_pte_valid(pte) && !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN);
}

static u64 stage2_map_walker_phys_addr(const struct kvm_pgtable_visit_ctx *ctx,
                                       const struct stage2_map_data *data)
{
        u64 phys = data->phys;

        /* Work out the correct PA based on how far the walk has gotten */
        return phys + (ctx->addr - ctx->start);
}

static bool stage2_leaf_mapping_allowed(const struct kvm_pgtable_visit_ctx *ctx,
                                        struct stage2_map_data *data)
{
        u64 phys = stage2_map_walker_phys_addr(ctx, data);

        if (data->force_pte && ctx->level < KVM_PGTABLE_LAST_LEVEL)
                return false;

        if (data->annotation)
                return true;

        return kvm_block_mapping_supported(ctx, phys);
}

static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
                                      struct stage2_map_data *data)
{
        kvm_pte_t new;
        u64 phys = stage2_map_walker_phys_addr(ctx, data);
        u64 granule = kvm_granule_size(ctx->level);
        struct kvm_pgtable *pgt = data->mmu->pgt;
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (!stage2_leaf_mapping_allowed(ctx, data))
                return -E2BIG;

        if (!data->annotation)
                new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
        else
                new = kvm_init_invalid_leaf_owner(data->owner_id);

        /*
         * Skip updating the PTE if we are trying to recreate the exact
         * same mapping or only change the access permissions. Instead,
         * the vCPU will exit one more time from guest if still needed
         * and then go through the path of relaxing permissions.
         */
        if (!stage2_pte_needs_update(ctx->old, new))
                return -EAGAIN;

        /* If we're only changing software bits, then store them and go! */
        if (!kvm_pgtable_walk_shared(ctx) &&
            !((ctx->old ^ new) & ~KVM_PTE_LEAF_ATTR_HI_SW)) {
                bool old_is_counted = stage2_pte_is_counted(ctx->old);

                if (old_is_counted != stage2_pte_is_counted(new)) {
                        if (old_is_counted)
                                mm_ops->put_page(ctx->ptep);
                        else
                                mm_ops->get_page(ctx->ptep);
                }
                WARN_ON_ONCE(!stage2_try_set_pte(ctx, new));
                return 0;
        }

        if (!stage2_try_break_pte(ctx, data->mmu))
                return -EAGAIN;

        /* Perform CMOs before installation of the guest stage-2 PTE */
        if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->dcache_clean_inval_poc &&
            stage2_pte_cacheable(pgt, new))
                mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops),
                                               granule);

        if (!kvm_pgtable_walk_skip_cmo(ctx) && mm_ops->icache_inval_pou &&
            stage2_pte_executable(new))
                mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule);

        stage2_make_pte(ctx, new);

        return 0;
}

static int stage2_map_walk_table_pre(const struct kvm_pgtable_visit_ctx *ctx,
                                     struct stage2_map_data *data)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
        kvm_pte_t *childp = kvm_pte_follow(ctx->old, mm_ops);
        int ret;

        if (!stage2_leaf_mapping_allowed(ctx, data))
                return 0;

        ret = stage2_map_walker_try_leaf(ctx, data);
        if (ret)
                return ret;

        mm_ops->free_unlinked_table(childp, ctx->level);
        return 0;
}

static int stage2_map_walk_leaf(const struct kvm_pgtable_visit_ctx *ctx,
                                struct stage2_map_data *data)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
        kvm_pte_t *childp, new;
        int ret;

        ret = stage2_map_walker_try_leaf(ctx, data);
        if (ret != -E2BIG)
                return ret;

        if (WARN_ON(ctx->level == KVM_PGTABLE_LAST_LEVEL))
                return -EINVAL;

        if (!data->memcache)
                return -ENOMEM;

        childp = mm_ops->zalloc_page(data->memcache);
        if (!childp)
                return -ENOMEM;

        if (!stage2_try_break_pte(ctx, data->mmu)) {
                mm_ops->put_page(childp);
                return -EAGAIN;
        }

        /*
         * If we've run into an existing block mapping then replace it with
         * a table. Accesses beyond 'end' that fall within the new table
         * will be mapped lazily.
         */
        new = kvm_init_table_pte(childp, mm_ops);
        stage2_make_pte(ctx, new);

        return 0;
}

/*
 * The TABLE_PRE callback runs for table entries on the way down, looking
 * for table entries which we could conceivably replace with a block entry
 * for this mapping. If it finds one it replaces the entry and calls
 * kvm_pgtable_mm_ops::free_unlinked_table() to tear down the detached table.
 *
 * Otherwise, the LEAF callback performs the mapping at the existing leaves
 * instead.
 */
static int stage2_map_walker(const struct kvm_pgtable_visit_ctx *ctx,
                             enum kvm_pgtable_walk_flags visit)
{
        struct stage2_map_data *data = ctx->arg;

        switch (visit) {
        case KVM_PGTABLE_WALK_TABLE_PRE:
                return stage2_map_walk_table_pre(ctx, data);
        case KVM_PGTABLE_WALK_LEAF:
                return stage2_map_walk_leaf(ctx, data);
        default:
                return -EINVAL;
        }
}

int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
                           u64 phys, enum kvm_pgtable_prot prot,
                           void *mc, enum kvm_pgtable_walk_flags flags)
{
        int ret;
        struct stage2_map_data map_data = {
                .phys                = ALIGN_DOWN(phys, PAGE_SIZE),
                .mmu                = pgt->mmu,
                .memcache        = mc,
                .force_pte        = pgt->force_pte_cb && pgt->force_pte_cb(addr, addr + size, prot),
        };
        struct kvm_pgtable_walker walker = {
                .cb                = stage2_map_walker,
                .flags                = flags |
                                  KVM_PGTABLE_WALK_TABLE_PRE |
                                  KVM_PGTABLE_WALK_LEAF,
                .arg                = &map_data,
        };

        if (WARN_ON((pgt->flags & KVM_PGTABLE_S2_IDMAP) && (addr != phys)))
                return -EINVAL;

        ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);
        if (ret)
                return ret;

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        dsb(ishst);
        return ret;
}

int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
                                 void *mc, u8 owner_id)
{
        int ret;
        struct stage2_map_data map_data = {
                .mmu                = pgt->mmu,
                .memcache        = mc,
                .owner_id        = owner_id,
                .force_pte        = true,
                .annotation        = true,
        };
        struct kvm_pgtable_walker walker = {
                .cb                = stage2_map_walker,
                .flags                = KVM_PGTABLE_WALK_TABLE_PRE |
                                  KVM_PGTABLE_WALK_LEAF,
                .arg                = &map_data,
        };

        if (owner_id > KVM_MAX_OWNER_ID)
                return -EINVAL;

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        return ret;
}

static int stage2_unmap_walker(const struct kvm_pgtable_visit_ctx *ctx,
                               enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable *pgt = ctx->arg;
        struct kvm_s2_mmu *mmu = pgt->mmu;
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
        kvm_pte_t *childp = NULL;
        bool need_flush = false;

        if (!kvm_pte_valid(ctx->old)) {
                if (stage2_pte_is_counted(ctx->old)) {
                        kvm_clear_pte(ctx->ptep);
                        mm_ops->put_page(ctx->ptep);
                }
                return 0;
        }

        if (kvm_pte_table(ctx->old, ctx->level)) {
                childp = kvm_pte_follow(ctx->old, mm_ops);

                if (mm_ops->page_count(childp) != 1)
                        return 0;
        } else if (stage2_pte_cacheable(pgt, ctx->old)) {
                need_flush = !stage2_has_fwb(pgt);
        }

        /*
         * This is similar to the map() path in that we unmap the entire
         * block entry and rely on the remaining portions being faulted
         * back lazily.
         */
        stage2_unmap_put_pte(ctx, mmu, mm_ops);

        if (need_flush && mm_ops->dcache_clean_inval_poc)
                mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
                                               kvm_granule_size(ctx->level));

        if (childp)
                mm_ops->put_page(childp);

        return 0;
}

int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
        int ret;
        struct kvm_pgtable_walker walker = {
                .cb        = stage2_unmap_walker,
                .arg        = pgt,
                .flags        = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
        };

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        if (stage2_unmap_defer_tlb_flush(pgt))
                /* Perform the deferred TLB invalidations */
                kvm_tlb_flush_vmid_range(pgt->mmu, addr, size);

        return ret;
}

struct stage2_attr_data {
        kvm_pte_t                        attr_set;
        kvm_pte_t                        attr_clr;
        kvm_pte_t                        pte;
        s8                                level;
};

static int stage2_attr_walker(const struct kvm_pgtable_visit_ctx *ctx,
                              enum kvm_pgtable_walk_flags visit)
{
        kvm_pte_t pte = ctx->old;
        struct stage2_attr_data *data = ctx->arg;
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (!kvm_pte_valid(ctx->old))
                return -EAGAIN;

        data->level = ctx->level;
        data->pte = pte;
        pte &= ~data->attr_clr;
        pte |= data->attr_set;

        /*
         * We may race with the CPU trying to set the access flag here,
         * but worst-case the access flag update gets lost and will be
         * set on the next access instead.
         */
        if (data->pte != pte) {
                /*
                 * Invalidate instruction cache before updating the guest
                 * stage-2 PTE if we are going to add executable permission.
                 */
                if (mm_ops->icache_inval_pou &&
                    stage2_pte_executable(pte) && !stage2_pte_executable(ctx->old))
                        mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops),
                                                  kvm_granule_size(ctx->level));

                if (!stage2_try_set_pte(ctx, pte))
                        return -EAGAIN;
        }

        return 0;
}

static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr,
                                    u64 size, kvm_pte_t attr_set,
                                    kvm_pte_t attr_clr, kvm_pte_t *orig_pte,
                                    s8 *level, enum kvm_pgtable_walk_flags flags)
{
        int ret;
        kvm_pte_t attr_mask = KVM_PTE_LEAF_ATTR_LO | KVM_PTE_LEAF_ATTR_HI;
        struct stage2_attr_data data = {
                .attr_set        = attr_set & attr_mask,
                .attr_clr        = attr_clr & attr_mask,
        };
        struct kvm_pgtable_walker walker = {
                .cb                = stage2_attr_walker,
                .arg                = &data,
                .flags                = flags | KVM_PGTABLE_WALK_LEAF,
        };

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        if (ret)
                return ret;

        if (orig_pte)
                *orig_pte = data.pte;

        if (level)
                *level = data.level;
        return 0;
}

int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
        return stage2_update_leaf_attrs(pgt, addr, size, 0,
                                        KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
                                        NULL, NULL, 0);
}

void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,
                                enum kvm_pgtable_walk_flags flags)
{
        int ret;

        ret = stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0,
                                       NULL, NULL, flags);
        if (!ret)
                dsb(ishst);
}

struct stage2_age_data {
        bool        mkold;
        bool        young;
};

static int stage2_age_walker(const struct kvm_pgtable_visit_ctx *ctx,
                             enum kvm_pgtable_walk_flags visit)
{
        kvm_pte_t new = ctx->old & ~KVM_PTE_LEAF_ATTR_LO_S2_AF;
        struct stage2_age_data *data = ctx->arg;

        if (!kvm_pte_valid(ctx->old) || new == ctx->old)
                return 0;

        data->young = true;

        /*
         * stage2_age_walker() is always called while holding the MMU lock for
         * write, so this will always succeed. Nonetheless, this deliberately
         * follows the race detection pattern of the other stage-2 walkers in
         * case the locking mechanics of the MMU notifiers is ever changed.
         */
        if (data->mkold && !stage2_try_set_pte(ctx, new))
                return -EAGAIN;

        /*
         * "But where's the TLBI?!", you scream.
         * "Over in the core code", I sigh.
         *
         * See the '->clear_flush_young()' callback on the KVM mmu notifier.
         */
        return 0;
}

bool kvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr,
                                         u64 size, bool mkold)
{
        struct stage2_age_data data = {
                .mkold                = mkold,
        };
        struct kvm_pgtable_walker walker = {
                .cb                = stage2_age_walker,
                .arg                = &data,
                .flags                = KVM_PGTABLE_WALK_LEAF,
        };

        WARN_ON(kvm_pgtable_walk(pgt, addr, size, &walker));
        return data.young;
}

int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr,
                                   enum kvm_pgtable_prot prot, enum kvm_pgtable_walk_flags flags)
{
        int ret;
        s8 level;
        kvm_pte_t set = 0, clr = 0;

        if (prot & KVM_PTE_LEAF_ATTR_HI_SW)
                return -EINVAL;

        if (prot & KVM_PGTABLE_PROT_R)
                set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_R;

        if (prot & KVM_PGTABLE_PROT_W)
                set |= KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W;

        if (prot & KVM_PGTABLE_PROT_X)
                clr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;

        ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level, flags);
        if (!ret || ret == -EAGAIN)
                kvm_call_hyp(__kvm_tlb_flush_vmid_ipa_nsh, pgt->mmu, addr, level);
        return ret;
}

static int stage2_flush_walker(const struct kvm_pgtable_visit_ctx *ctx,
                               enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable *pgt = ctx->arg;
        struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;

        if (!stage2_pte_cacheable(pgt, ctx->old))
                return 0;

        if (mm_ops->dcache_clean_inval_poc)
                mm_ops->dcache_clean_inval_poc(kvm_pte_follow(ctx->old, mm_ops),
                                               kvm_granule_size(ctx->level));
        return 0;
}

int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
        struct kvm_pgtable_walker walker = {
                .cb        = stage2_flush_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF,
                .arg        = pgt,
        };

        if (stage2_has_fwb(pgt))
                return 0;

        return kvm_pgtable_walk(pgt, addr, size, &walker);
}

kvm_pte_t *kvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt,
                                              u64 phys, s8 level,
                                              enum kvm_pgtable_prot prot,
                                              void *mc, bool force_pte)
{
        struct stage2_map_data map_data = {
                .phys                = phys,
                .mmu                = pgt->mmu,
                .memcache        = mc,
                .force_pte        = force_pte,
        };
        struct kvm_pgtable_walker walker = {
                .cb                = stage2_map_walker,
                .flags                = KVM_PGTABLE_WALK_LEAF |
                                  KVM_PGTABLE_WALK_SKIP_BBM_TLBI |
                                  KVM_PGTABLE_WALK_SKIP_CMO,
                .arg                = &map_data,
        };
        /*
         * The input address (.addr) is irrelevant for walking an
         * unlinked table. Construct an ambiguous IA range to map
         * kvm_granule_size(level) worth of memory.
         */
        struct kvm_pgtable_walk_data data = {
                .walker        = &walker,
                .addr        = 0,
                .end        = kvm_granule_size(level),
        };
        struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
        kvm_pte_t *pgtable;
        int ret;

        if (!IS_ALIGNED(phys, kvm_granule_size(level)))
                return ERR_PTR(-EINVAL);

        ret = stage2_set_prot_attr(pgt, prot, &map_data.attr);
        if (ret)
                return ERR_PTR(ret);

        pgtable = mm_ops->zalloc_page(mc);
        if (!pgtable)
                return ERR_PTR(-ENOMEM);

        ret = __kvm_pgtable_walk(&data, mm_ops, (kvm_pteref_t)pgtable,
                                 level + 1);
        if (ret) {
                kvm_pgtable_stage2_free_unlinked(mm_ops, pgtable, level);
                return ERR_PTR(ret);
        }

        return pgtable;
}

/*
 * Get the number of page-tables needed to replace a block with a
 * fully populated tree up to the PTE entries. Note that @level is
 * interpreted as in "level @level entry".
 */
static int stage2_block_get_nr_page_tables(s8 level)
{
        switch (level) {
        case 1:
                return PTRS_PER_PTE + 1;
        case 2:
                return 1;
        case 3:
                return 0;
        default:
                WARN_ON_ONCE(level < KVM_PGTABLE_MIN_BLOCK_LEVEL ||
                             level > KVM_PGTABLE_LAST_LEVEL);
                return -EINVAL;
        };
}

static int stage2_split_walker(const struct kvm_pgtable_visit_ctx *ctx,
                               enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
        struct kvm_mmu_memory_cache *mc = ctx->arg;
        struct kvm_s2_mmu *mmu;
        kvm_pte_t pte = ctx->old, new, *childp;
        enum kvm_pgtable_prot prot;
        s8 level = ctx->level;
        bool force_pte;
        int nr_pages;
        u64 phys;

        /* No huge-pages exist at the last level */
        if (level == KVM_PGTABLE_LAST_LEVEL)
                return 0;

        /* We only split valid block mappings */
        if (!kvm_pte_valid(pte))
                return 0;

        nr_pages = stage2_block_get_nr_page_tables(level);
        if (nr_pages < 0)
                return nr_pages;

        if (mc->nobjs >= nr_pages) {
                /* Build a tree mapped down to the PTE granularity. */
                force_pte = true;
        } else {
                /*
                 * Don't force PTEs, so create_unlinked() below does
                 * not populate the tree up to the PTE level. The
                 * consequence is that the call will require a single
                 * page of level 2 entries at level 1, or a single
                 * page of PTEs at level 2. If we are at level 1, the
                 * PTEs will be created recursively.
                 */
                force_pte = false;
                nr_pages = 1;
        }

        if (mc->nobjs < nr_pages)
                return -ENOMEM;

        mmu = container_of(mc, struct kvm_s2_mmu, split_page_cache);
        phys = kvm_pte_to_phys(pte);
        prot = kvm_pgtable_stage2_pte_prot(pte);

        childp = kvm_pgtable_stage2_create_unlinked(mmu->pgt, phys,
                                                    level, prot, mc, force_pte);
        if (IS_ERR(childp))
                return PTR_ERR(childp);

        if (!stage2_try_break_pte(ctx, mmu)) {
                kvm_pgtable_stage2_free_unlinked(mm_ops, childp, level);
                return -EAGAIN;
        }

        /*
         * Note, the contents of the page table are guaranteed to be made
         * visible before the new PTE is assigned because stage2_make_pte()
         * writes the PTE using smp_store_release().
         */
        new = kvm_init_table_pte(childp, mm_ops);
        stage2_make_pte(ctx, new);
        return 0;
}

int kvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,
                             struct kvm_mmu_memory_cache *mc)
{
        struct kvm_pgtable_walker walker = {
                .cb        = stage2_split_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF,
                .arg        = mc,
        };
        int ret;

        ret = kvm_pgtable_walk(pgt, addr, size, &walker);
        dsb(ishst);
        return ret;
}

int __kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
                              struct kvm_pgtable_mm_ops *mm_ops,
                              enum kvm_pgtable_stage2_flags flags,
                              kvm_pgtable_force_pte_cb_t force_pte_cb)
{
        size_t pgd_sz;
        u64 vtcr = mmu->vtcr;
        u32 ia_bits = VTCR_EL2_IPA(vtcr);
        u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
        s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;

        pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
        pgt->pgd = (kvm_pteref_t)mm_ops->zalloc_pages_exact(pgd_sz);
        if (!pgt->pgd)
                return -ENOMEM;

        pgt->ia_bits                = ia_bits;
        pgt->start_level        = start_level;
        pgt->mm_ops                = mm_ops;
        pgt->mmu                = mmu;
        pgt->flags                = flags;
        pgt->force_pte_cb        = force_pte_cb;

        /* Ensure zeroed PGD pages are visible to the hardware walker */
        dsb(ishst);
        return 0;
}

size_t kvm_pgtable_stage2_pgd_size(u64 vtcr)
{
        u32 ia_bits = VTCR_EL2_IPA(vtcr);
        u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
        s8 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;

        return kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
}

static int stage2_free_walker(const struct kvm_pgtable_visit_ctx *ctx,
                              enum kvm_pgtable_walk_flags visit)
{
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;

        if (!stage2_pte_is_counted(ctx->old))
                return 0;

        mm_ops->put_page(ctx->ptep);

        if (kvm_pte_table(ctx->old, ctx->level))
                mm_ops->put_page(kvm_pte_follow(ctx->old, mm_ops));

        return 0;
}

void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)
{
        size_t pgd_sz;
        struct kvm_pgtable_walker walker = {
                .cb        = stage2_free_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF |
                          KVM_PGTABLE_WALK_TABLE_POST,
        };

        WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
        pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
        pgt->mm_ops->free_pages_exact(kvm_dereference_pteref(&walker, pgt->pgd), pgd_sz);
        pgt->pgd = NULL;
}

void kvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level)
{
        kvm_pteref_t ptep = (kvm_pteref_t)pgtable;
        struct kvm_pgtable_walker walker = {
                .cb        = stage2_free_walker,
                .flags        = KVM_PGTABLE_WALK_LEAF |
                          KVM_PGTABLE_WALK_TABLE_POST,
        };
        struct kvm_pgtable_walk_data data = {
                .walker        = &walker,

                /*
                 * At this point the IPA really doesn't matter, as the page
                 * table being traversed has already been removed from the stage
                 * 2. Set an appropriate range to cover the entire page table.
                 */
                .addr        = 0,
                .end        = kvm_granule_size(level),
        };

        WARN_ON(__kvm_pgtable_walk(&data, mm_ops, ptep, level + 1));

        WARN_ON(mm_ops->page_count(pgtable) != 1);
        mm_ops->put_page(pgtable);
}






























  205 













  258 
   44 










   44 
















  158 

















  258 

   38 


  205 







































































  158 







  257 


  257 



  256 







  205 






  205 





  158 

  158 

































  199 


  139 
  102 









  204 







  205 

   38 


  205 



  204 






  205 





  159 



   20 


  158 
  158 


  157 
  157 



  158 









































  205 

  205 


   38 
   38 
   38 




















  158 

  158 




   20 
   20 












  318 




















  243 









  318 





   22 






























  316 
















































  318 








































































































  587 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_MM_INLINE_H
#define LINUX_MM_INLINE_H

#include <linux/atomic.h>
#include <linux/huge_mm.h>
#include <linux/mm_types.h>
#include <linux/swap.h>
#include <linux/string.h>
#include <linux/userfaultfd_k.h>
#include <linux/swapops.h>

/**
 * folio_is_file_lru - Should the folio be on a file LRU or anon LRU?
 * @folio: The folio to test.
 *
 * We would like to get this info without a page flag, but the state
 * needs to survive until the folio is last deleted from the LRU, which
 * could be as far down as __page_cache_release.
 *
 * Return: An integer (not a boolean!) used to sort a folio onto the
 * right LRU list and to account folios correctly.
 * 1 if @folio is a regular filesystem backed page cache folio
 * or a lazily freed anonymous folio (e.g. via MADV_FREE).
 * 0 if @folio is a normal anonymous folio, a tmpfs folio or otherwise
 * ram or swap backed folio.
 */
static inline int folio_is_file_lru(struct folio *folio)
{
        return !folio_test_swapbacked(folio);
}

static inline int page_is_file_lru(struct page *page)
{
        return folio_is_file_lru(page_folio(page));
}

static __always_inline void __update_lru_size(struct lruvec *lruvec,
                                enum lru_list lru, enum zone_type zid,
                                long nr_pages)
{
        struct pglist_data *pgdat = lruvec_pgdat(lruvec);

        lockdep_assert_held(&lruvec->lru_lock);
        WARN_ON_ONCE(nr_pages != (int)nr_pages);

        __mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
        __mod_zone_page_state(&pgdat->node_zones[zid],
                                NR_ZONE_LRU_BASE + lru, nr_pages);
}

static __always_inline void update_lru_size(struct lruvec *lruvec,
                                enum lru_list lru, enum zone_type zid,
                                long nr_pages)
{
        __update_lru_size(lruvec, lru, zid, nr_pages);
#ifdef CONFIG_MEMCG
        mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
#endif
}

/**
 * __folio_clear_lru_flags - Clear page lru flags before releasing a page.
 * @folio: The folio that was on lru and now has a zero reference.
 */
static __always_inline void __folio_clear_lru_flags(struct folio *folio)
{
        VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);

        __folio_clear_lru(folio);

        /* this shouldn't happen, so leave the flags to bad_page() */
        if (folio_test_active(folio) && folio_test_unevictable(folio))
                return;

        __folio_clear_active(folio);
        __folio_clear_unevictable(folio);
}

/**
 * folio_lru_list - Which LRU list should a folio be on?
 * @folio: The folio to test.
 *
 * Return: The LRU list a folio should be on, as an index
 * into the array of LRU lists.
 */
static __always_inline enum lru_list folio_lru_list(struct folio *folio)
{
        enum lru_list lru;

        VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);

        if (folio_test_unevictable(folio))
                return LRU_UNEVICTABLE;

        lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;
        if (folio_test_active(folio))
                lru += LRU_ACTIVE;

        return lru;
}

#ifdef CONFIG_LRU_GEN

#ifdef CONFIG_LRU_GEN_ENABLED
static inline bool lru_gen_enabled(void)
{
        DECLARE_STATIC_KEY_TRUE(lru_gen_caps[NR_LRU_GEN_CAPS]);

        return static_branch_likely(&lru_gen_caps[LRU_GEN_CORE]);
}
#else
static inline bool lru_gen_enabled(void)
{
        DECLARE_STATIC_KEY_FALSE(lru_gen_caps[NR_LRU_GEN_CAPS]);

        return static_branch_unlikely(&lru_gen_caps[LRU_GEN_CORE]);
}
#endif

static inline bool lru_gen_in_fault(void)
{
        return current->in_lru_fault;
}

static inline int lru_gen_from_seq(unsigned long seq)
{
        return seq % MAX_NR_GENS;
}

static inline int lru_hist_from_seq(unsigned long seq)
{
        return seq % NR_HIST_GENS;
}

static inline int lru_tier_from_refs(int refs, bool workingset)
{
        VM_WARN_ON_ONCE(refs > BIT(LRU_REFS_WIDTH));

        /* see the comment on MAX_NR_TIERS */
        return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs);
}

static inline int folio_lru_refs(struct folio *folio)
{
        unsigned long flags = READ_ONCE(folio->flags);

        if (!(flags & BIT(PG_referenced)))
                return 0;
        /*
         * Return the total number of accesses including PG_referenced. Also see
         * the comment on LRU_REFS_FLAGS.
         */
        return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1;
}

static inline int folio_lru_gen(struct folio *folio)
{
        unsigned long flags = READ_ONCE(folio->flags);

        return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;
}

static inline bool lru_gen_is_active(struct lruvec *lruvec, int gen)
{
        unsigned long max_seq = lruvec->lrugen.max_seq;

        VM_WARN_ON_ONCE(gen >= MAX_NR_GENS);

        /* see the comment on MIN_NR_GENS */
        return gen == lru_gen_from_seq(max_seq) || gen == lru_gen_from_seq(max_seq - 1);
}

static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
                                       int old_gen, int new_gen)
{
        int type = folio_is_file_lru(folio);
        int zone = folio_zonenum(folio);
        int delta = folio_nr_pages(folio);
        enum lru_list lru = type * LRU_INACTIVE_FILE;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;

        VM_WARN_ON_ONCE(old_gen != -1 && old_gen >= MAX_NR_GENS);
        VM_WARN_ON_ONCE(new_gen != -1 && new_gen >= MAX_NR_GENS);
        VM_WARN_ON_ONCE(old_gen == -1 && new_gen == -1);

        if (old_gen >= 0)
                WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone],
                           lrugen->nr_pages[old_gen][type][zone] - delta);
        if (new_gen >= 0)
                WRITE_ONCE(lrugen->nr_pages[new_gen][type][zone],
                           lrugen->nr_pages[new_gen][type][zone] + delta);

        /* addition */
        if (old_gen < 0) {
                if (lru_gen_is_active(lruvec, new_gen))
                        lru += LRU_ACTIVE;
                __update_lru_size(lruvec, lru, zone, delta);
                return;
        }

        /* deletion */
        if (new_gen < 0) {
                if (lru_gen_is_active(lruvec, old_gen))
                        lru += LRU_ACTIVE;
                __update_lru_size(lruvec, lru, zone, -delta);
                return;
        }

        /* promotion */
        if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
                __update_lru_size(lruvec, lru, zone, -delta);
                __update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
        }

        /* demotion requires isolation, e.g., lru_deactivate_fn() */
        VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
}

static inline unsigned long lru_gen_folio_seq(struct lruvec *lruvec, struct folio *folio,
                                              bool reclaiming)
{
        int gen;
        int type = folio_is_file_lru(folio);
        struct lru_gen_folio *lrugen = &lruvec->lrugen;

        /*
         * +-----------------------------------+-----------------------------------+
         * | Accessed through page tables and  | Accessed through file descriptors |
         * | promoted by folio_update_gen()    | and protected by folio_inc_gen()  |
         * +-----------------------------------+-----------------------------------+
         * | PG_active (set while isolated)    |                                   |
         * +-----------------+-----------------+-----------------+-----------------+
         * |  PG_workingset  |  PG_referenced  |  PG_workingset  |  LRU_REFS_FLAGS |
         * +-----------------------------------+-----------------------------------+
         * |<---------- MIN_NR_GENS ---------->|                                   |
         * |<---------------------------- MAX_NR_GENS ---------------------------->|
         */
        if (folio_test_active(folio))
                gen = MIN_NR_GENS - folio_test_workingset(folio);
        else if (reclaiming)
                gen = MAX_NR_GENS;
        else if ((!folio_is_file_lru(folio) && !folio_test_swapcache(folio)) ||
                 (folio_test_reclaim(folio) &&
                  (folio_test_dirty(folio) || folio_test_writeback(folio))))
                gen = MIN_NR_GENS;
        else
                gen = MAX_NR_GENS - folio_test_workingset(folio);

        return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type]));
}

static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
        unsigned long seq;
        unsigned long flags;
        int gen = folio_lru_gen(folio);
        int type = folio_is_file_lru(folio);
        int zone = folio_zonenum(folio);
        struct lru_gen_folio *lrugen = &lruvec->lrugen;

        VM_WARN_ON_ONCE_FOLIO(gen != -1, folio);

        if (folio_test_unevictable(folio) || !lrugen->enabled)
                return false;

        seq = lru_gen_folio_seq(lruvec, folio, reclaiming);
        gen = lru_gen_from_seq(seq);
        flags = (gen + 1UL) << LRU_GEN_PGOFF;
        /* see the comment on MIN_NR_GENS about PG_active */
        set_mask_bits(&folio->flags, LRU_GEN_MASK | BIT(PG_active), flags);

        lru_gen_update_size(lruvec, folio, -1, gen);
        /* for folio_rotate_reclaimable() */
        if (reclaiming)
                list_add_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
        else
                list_add(&folio->lru, &lrugen->folios[gen][type][zone]);

        return true;
}

static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
        unsigned long flags;
        int gen = folio_lru_gen(folio);

        if (gen < 0)
                return false;

        VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
        VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);

        /* for folio_migrate_flags() */
        flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
        flags = set_mask_bits(&folio->flags, LRU_GEN_MASK, flags);
        gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1;

        lru_gen_update_size(lruvec, folio, gen, -1);
        list_del(&folio->lru);

        return true;
}

static inline void folio_migrate_refs(struct folio *new, struct folio *old)
{
        unsigned long refs = READ_ONCE(old->flags) & LRU_REFS_MASK;

        set_mask_bits(&new->flags, LRU_REFS_MASK, refs);
}
#else /* !CONFIG_LRU_GEN */

static inline bool lru_gen_enabled(void)
{
        return false;
}

static inline bool lru_gen_in_fault(void)
{
        return false;
}

static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
        return false;
}

static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
{
        return false;
}

static inline void folio_migrate_refs(struct folio *new, struct folio *old)
{

}
#endif /* CONFIG_LRU_GEN */

static __always_inline
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
{
        enum lru_list lru = folio_lru_list(folio);

        if (lru_gen_add_folio(lruvec, folio, false))
                return;

        update_lru_size(lruvec, lru, folio_zonenum(folio),
                        folio_nr_pages(folio));
        if (lru != LRU_UNEVICTABLE)
                list_add(&folio->lru, &lruvec->lists[lru]);
}

static __always_inline
void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
{
        enum lru_list lru = folio_lru_list(folio);

        if (lru_gen_add_folio(lruvec, folio, true))
                return;

        update_lru_size(lruvec, lru, folio_zonenum(folio),
                        folio_nr_pages(folio));
        /* This is not expected to be used on LRU_UNEVICTABLE */
        list_add_tail(&folio->lru, &lruvec->lists[lru]);
}

static __always_inline
void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
{
        enum lru_list lru = folio_lru_list(folio);

        if (lru_gen_del_folio(lruvec, folio, false))
                return;

        if (lru != LRU_UNEVICTABLE)
                list_del(&folio->lru);
        update_lru_size(lruvec, lru, folio_zonenum(folio),
                        -folio_nr_pages(folio));
}

#ifdef CONFIG_ANON_VMA_NAME
/* mmap_lock should be read-locked */
static inline void anon_vma_name_get(struct anon_vma_name *anon_name)
{
        if (anon_name)
                kref_get(&anon_name->kref);
}

static inline void anon_vma_name_put(struct anon_vma_name *anon_name)
{
        if (anon_name)
                kref_put(&anon_name->kref, anon_vma_name_free);
}

static inline
struct anon_vma_name *anon_vma_name_reuse(struct anon_vma_name *anon_name)
{
        /* Prevent anon_name refcount saturation early on */
        if (kref_read(&anon_name->kref) < REFCOUNT_MAX) {
                anon_vma_name_get(anon_name);
                return anon_name;

        }
        return anon_vma_name_alloc(anon_name->name);
}

static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
                                     struct vm_area_struct *new_vma)
{
        struct anon_vma_name *anon_name = anon_vma_name(orig_vma);

        if (anon_name)
                new_vma->anon_name = anon_vma_name_reuse(anon_name);
}

static inline void free_anon_vma_name(struct vm_area_struct *vma)
{
        /*
         * Not using anon_vma_name because it generates a warning if mmap_lock
         * is not held, which might be the case here.
         */
        anon_vma_name_put(vma->anon_name);
}

static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
                                    struct anon_vma_name *anon_name2)
{
        if (anon_name1 == anon_name2)
                return true;

        return anon_name1 && anon_name2 &&
                !strcmp(anon_name1->name, anon_name2->name);
}

#else /* CONFIG_ANON_VMA_NAME */
static inline void anon_vma_name_get(struct anon_vma_name *anon_name) {}
static inline void anon_vma_name_put(struct anon_vma_name *anon_name) {}
static inline void dup_anon_vma_name(struct vm_area_struct *orig_vma,
                                     struct vm_area_struct *new_vma) {}
static inline void free_anon_vma_name(struct vm_area_struct *vma) {}

static inline bool anon_vma_name_eq(struct anon_vma_name *anon_name1,
                                    struct anon_vma_name *anon_name2)
{
        return true;
}

#endif  /* CONFIG_ANON_VMA_NAME */

void pfnmap_track_ctx_release(struct kref *ref);

static inline void init_tlb_flush_pending(struct mm_struct *mm)
{
        atomic_set(&mm->tlb_flush_pending, 0);
}

static inline void inc_tlb_flush_pending(struct mm_struct *mm)
{
        atomic_inc(&mm->tlb_flush_pending);
        /*
         * The only time this value is relevant is when there are indeed pages
         * to flush. And we'll only flush pages after changing them, which
         * requires the PTL.
         *
         * So the ordering here is:
         *
         *        atomic_inc(&mm->tlb_flush_pending);
         *        spin_lock(&ptl);
         *        ...
         *        set_pte_at();
         *        spin_unlock(&ptl);
         *
         *                                spin_lock(&ptl)
         *                                mm_tlb_flush_pending();
         *                                ....
         *                                spin_unlock(&ptl);
         *
         *        flush_tlb_range();
         *        atomic_dec(&mm->tlb_flush_pending);
         *
         * Where the increment if constrained by the PTL unlock, it thus
         * ensures that the increment is visible if the PTE modification is
         * visible. After all, if there is no PTE modification, nobody cares
         * about TLB flushes either.
         *
         * This very much relies on users (mm_tlb_flush_pending() and
         * mm_tlb_flush_nested()) only caring about _specific_ PTEs (and
         * therefore specific PTLs), because with SPLIT_PTE_PTLOCKS and RCpc
         * locks (PPC) the unlock of one doesn't order against the lock of
         * another PTL.
         *
         * The decrement is ordered by the flush_tlb_range(), such that
         * mm_tlb_flush_pending() will not return false unless all flushes have
         * completed.
         */
}

static inline void dec_tlb_flush_pending(struct mm_struct *mm)
{
        /*
         * See inc_tlb_flush_pending().
         *
         * This cannot be smp_mb__before_atomic() because smp_mb() simply does
         * not order against TLB invalidate completion, which is what we need.
         *
         * Therefore we must rely on tlb_flush_*() to guarantee order.
         */
        atomic_dec(&mm->tlb_flush_pending);
}

static inline bool mm_tlb_flush_pending(struct mm_struct *mm)
{
        /*
         * Must be called after having acquired the PTL; orders against that
         * PTLs release and therefore ensures that if we observe the modified
         * PTE we must also observe the increment from inc_tlb_flush_pending().
         *
         * That is, it only guarantees to return true if there is a flush
         * pending for _this_ PTL.
         */
        return atomic_read(&mm->tlb_flush_pending);
}

static inline bool mm_tlb_flush_nested(struct mm_struct *mm)
{
        /*
         * Similar to mm_tlb_flush_pending(), we must have acquired the PTL
         * for which there is a TLB flush pending in order to guarantee
         * we've seen both that PTE modification and the increment.
         *
         * (no requirement on actually still holding the PTL, that is irrelevant)
         */
        return atomic_read(&mm->tlb_flush_pending) > 1;
}

#ifdef CONFIG_MMU
/*
 * Computes the pte marker to copy from the given source entry into dst_vma.
 * If no marker should be copied, returns 0.
 * The caller should insert a new pte created with make_pte_marker().
 */
static inline pte_marker copy_pte_marker(
                swp_entry_t entry, struct vm_area_struct *dst_vma)
{
        pte_marker srcm = pte_marker_get(entry);
        /* Always copy error entries. */
        pte_marker dstm = srcm & (PTE_MARKER_POISONED | PTE_MARKER_GUARD);

        /* Only copy PTE markers if UFFD register matches. */
        if ((srcm & PTE_MARKER_UFFD_WP) && userfaultfd_wp(dst_vma))
                dstm |= PTE_MARKER_UFFD_WP;

        return dstm;
}
#endif

/*
 * If this pte is wr-protected by uffd-wp in any form, arm the special pte to
 * replace a none pte.  NOTE!  This should only be called when *pte is already
 * cleared so we will never accidentally replace something valuable.  Meanwhile
 * none pte also means we are not demoting the pte so tlb flushed is not needed.
 * E.g., when pte cleared the caller should have taken care of the tlb flush.
 *
 * Must be called with pgtable lock held so that no thread will see the none
 * pte, and if they see it, they'll fault and serialize at the pgtable lock.
 *
 * Returns true if an uffd-wp pte was installed, false otherwise.
 */
static inline bool
pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
                              pte_t *pte, pte_t pteval)
{
#ifdef CONFIG_PTE_MARKER_UFFD_WP
        bool arm_uffd_pte = false;

        /* The current status of the pte should be "cleared" before calling */
        WARN_ON_ONCE(!pte_none(ptep_get(pte)));

        /*
         * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole
         * thing, because when zapping either it means it's dropping the
         * page, or in TTU where the present pte will be quickly replaced
         * with a swap pte.  There's no way of leaking the bit.
         */
        if (vma_is_anonymous(vma) || !userfaultfd_wp(vma))
                return false;

        /* A uffd-wp wr-protected normal pte */
        if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval)))
                arm_uffd_pte = true;

        /*
         * A uffd-wp wr-protected swap pte.  Note: this should even cover an
         * existing pte marker with uffd-wp bit set.
         */
        if (unlikely(pte_swp_uffd_wp_any(pteval)))
                arm_uffd_pte = true;

        if (unlikely(arm_uffd_pte)) {
                set_pte_at(vma->vm_mm, addr, pte,
                           make_pte_marker(PTE_MARKER_UFFD_WP));
                return true;
        }
#endif
        return false;
}

static inline bool vma_has_recency(struct vm_area_struct *vma)
{
        if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))
                return false;

        if (vma->vm_file && (vma->vm_file->f_mode & FMODE_NOREUSE))
                return false;

        return true;
}

#endif

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_BITOPS_BUILTIN___FFS_H_
#define _ASM_GENERIC_BITOPS_BUILTIN___FFS_H_

/**
 * __ffs - find first bit in word.
 * @word: The word to search
 *
 * Undefined if no bit exists, so code should check against 0 first.
 */
static __always_inline unsigned int __ffs(unsigned long word)
{
        return __builtin_ctzl(word);
}

#endif





































































   17 



















  462 


  206 































  463 
  461 











































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Access vector cache interface for object managers.
 *
 * Author : Stephen Smalley, <stephen.smalley.work@gmail.com>
 */

#ifndef _SELINUX_AVC_H_
#define _SELINUX_AVC_H_

#include <linux/stddef.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kdev_t.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/audit.h>
#include <linux/lsm_audit.h>
#include <linux/in6.h>
#include "flask.h"
#include "av_permissions.h"
#include "security.h"

/*
 * An entry in the AVC.
 */
struct avc_entry;

struct task_struct;
struct inode;
struct sock;
struct sk_buff;

/*
 * AVC statistics
 */
struct avc_cache_stats {
        unsigned int lookups;
        unsigned int misses;
        unsigned int allocations;
        unsigned int reclaims;
        unsigned int frees;
};

/*
 * We only need this data after we have decided to send an audit message.
 */
struct selinux_audit_data {
        u32 ssid;
        u32 tsid;
        u16 tclass;
        u32 requested;
        u32 audited;
        u32 denied;
        int result;
} __randomize_layout;

/*
 * AVC operations
 */

void __init avc_init(void);

static inline u32 avc_audit_required(u32 requested, struct av_decision *avd,
                                     int result, u32 auditdeny, u32 *deniedp)
{
        u32 denied, audited;
        denied = requested & ~avd->allowed;
        if (unlikely(denied)) {
                audited = denied & avd->auditdeny;
                /*
                 * auditdeny is TRICKY!  Setting a bit in
                 * this field means that ANY denials should NOT be audited if
                 * the policy contains an explicit dontaudit rule for that
                 * permission.  Take notice that this is unrelated to the
                 * actual permissions that were denied.  As an example lets
                 * assume:
                 *
                 * denied == READ
                 * avd.auditdeny & ACCESS == 0 (not set means explicit rule)
                 * auditdeny & ACCESS == 1
                 *
                 * We will NOT audit the denial even though the denied
                 * permission was READ and the auditdeny checks were for
                 * ACCESS
                 */
                if (auditdeny && !(auditdeny & avd->auditdeny))
                        audited = 0;
        } else if (result)
                audited = denied = requested;
        else
                audited = requested & avd->auditallow;
        *deniedp = denied;
        return audited;
}

int slow_avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested, u32 audited,
                   u32 denied, int result, struct common_audit_data *a);

/**
 * avc_audit - Audit the granting or denial of permissions.
 * @ssid: source security identifier
 * @tsid: target security identifier
 * @tclass: target security class
 * @requested: requested permissions
 * @avd: access vector decisions
 * @result: result from avc_has_perm_noaudit
 * @a:  auxiliary audit data
 *
 * Audit the granting or denial of permissions in accordance
 * with the policy.  This function is typically called by
 * avc_has_perm() after a permission check, but can also be
 * called directly by callers who use avc_has_perm_noaudit()
 * in order to separate the permission check from the auditing.
 * For example, this separation is useful when the permission check must
 * be performed under a lock, to allow the lock to be released
 * before calling the auditing code.
 */
static inline int avc_audit(u32 ssid, u32 tsid, u16 tclass, u32 requested,
                            struct av_decision *avd, int result,
                            struct common_audit_data *a)
{
        u32 audited, denied;
        audited = avc_audit_required(requested, avd, result, 0, &denied);
        if (likely(!audited))
                return 0;
        return slow_avc_audit(ssid, tsid, tclass, requested, audited, denied,
                              result, a);
}

#define AVC_STRICT           1 /* Ignore permissive mode. */
#define AVC_EXTENDED_PERMS 2 /* update extended permissions */
int avc_has_perm_noaudit(u32 ssid, u32 tsid, u16 tclass, u32 requested,
                         unsigned int flags, struct av_decision *avd);

int avc_has_perm(u32 ssid, u32 tsid, u16 tclass, u32 requested,
                 struct common_audit_data *auditdata);

#define AVC_EXT_IOCTL        (1 << 0) /* Cache entry for an ioctl extended permission */
#define AVC_EXT_NLMSG        (1 << 1) /* Cache entry for an nlmsg extended permission */
int avc_has_extended_perms(u32 ssid, u32 tsid, u16 tclass, u32 requested,
                           u8 driver, u8 base_perm, u8 perm,
                           struct common_audit_data *ad);

u32 avc_policy_seqno(void);

#define AVC_CALLBACK_GRANT                1
#define AVC_CALLBACK_TRY_REVOKE                2
#define AVC_CALLBACK_REVOKE                4
#define AVC_CALLBACK_RESET                8
#define AVC_CALLBACK_AUDITALLOW_ENABLE        16
#define AVC_CALLBACK_AUDITALLOW_DISABLE 32
#define AVC_CALLBACK_AUDITDENY_ENABLE        64
#define AVC_CALLBACK_AUDITDENY_DISABLE        128
#define AVC_CALLBACK_ADD_XPERMS                256

int avc_add_callback(int (*callback)(u32 event), u32 events);

/* Exported to selinuxfs */
int avc_get_hash_stats(char *page);
unsigned int avc_get_cache_threshold(void);
void avc_set_cache_threshold(unsigned int cache_threshold);

#ifdef CONFIG_SECURITY_SELINUX_AVC_STATS
DECLARE_PER_CPU(struct avc_cache_stats, avc_cache_stats);
#endif

#endif /* _SELINUX_AVC_H_ */

































































































































































































































  147 
  147 






















  145 




  145 
  145 



  144 



  145 
   14 

    3 
  145 

























































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/mm/swap_state.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *
 *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
 */
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/kernel_stat.h>
#include <linux/mempolicy.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/migrate.h>
#include <linux/vmalloc.h>
#include <linux/huge_mm.h>
#include <linux/shmem_fs.h>
#include "internal.h"
#include "swap.h"

/*
 * swapper_space is a fiction, retained to simplify the path through
 * vmscan's shrink_folio_list.
 */
static const struct address_space_operations swap_aops = {
        .dirty_folio        = noop_dirty_folio,
#ifdef CONFIG_MIGRATION
        .migrate_folio        = migrate_folio,
#endif
};

struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly;
static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly;
static bool enable_vma_readahead __read_mostly = true;

#define SWAP_RA_ORDER_CEILING        5

#define SWAP_RA_WIN_SHIFT        (PAGE_SHIFT / 2)
#define SWAP_RA_HITS_MASK        ((1UL << SWAP_RA_WIN_SHIFT) - 1)
#define SWAP_RA_HITS_MAX        SWAP_RA_HITS_MASK
#define SWAP_RA_WIN_MASK        (~PAGE_MASK & ~SWAP_RA_HITS_MASK)

#define SWAP_RA_HITS(v)                ((v) & SWAP_RA_HITS_MASK)
#define SWAP_RA_WIN(v)                (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT)
#define SWAP_RA_ADDR(v)                ((v) & PAGE_MASK)

#define SWAP_RA_VAL(addr, win, hits)                                \
        (((addr) & PAGE_MASK) |                                        \
         (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) |        \
         ((hits) & SWAP_RA_HITS_MASK))

/* Initial readahead hits is 4 to start up with a small window */
#define GET_SWAP_RA_VAL(vma)                                        \
        (atomic_long_read(&(vma)->swap_readahead_info) ? : 4)

static atomic_t swapin_readahead_hits = ATOMIC_INIT(4);

void show_swap_cache_info(void)
{
        printk("%lu pages in swap cache\n", total_swapcache_pages());
        printk("Free swap  = %ldkB\n", K(get_nr_swap_pages()));
        printk("Total swap = %lukB\n", K(total_swap_pages));
}

void *get_shadow_from_swap_cache(swp_entry_t entry)
{
        struct address_space *address_space = swap_address_space(entry);
        pgoff_t idx = swap_cache_index(entry);
        void *shadow;

        shadow = xa_load(&address_space->i_pages, idx);
        if (xa_is_value(shadow))
                return shadow;
        return NULL;
}

/*
 * add_to_swap_cache resembles filemap_add_folio on swapper_space,
 * but sets SwapCache flag and 'swap' instead of mapping and index.
 */
int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
                        gfp_t gfp, void **shadowp)
{
        struct address_space *address_space = swap_address_space(entry);
        pgoff_t idx = swap_cache_index(entry);
        XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio));
        unsigned long i, nr = folio_nr_pages(folio);
        void *old;

        xas_set_update(&xas, workingset_update_node);

        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);

        folio_ref_add(folio, nr);
        folio_set_swapcache(folio);
        folio->swap = entry;

        do {
                xas_lock_irq(&xas);
                xas_create_range(&xas);
                if (xas_error(&xas))
                        goto unlock;
                for (i = 0; i < nr; i++) {
                        VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio);
                        if (shadowp) {
                                old = xas_load(&xas);
                                if (xa_is_value(old))
                                        *shadowp = old;
                        }
                        xas_store(&xas, folio);
                        xas_next(&xas);
                }
                address_space->nrpages += nr;
                __node_stat_mod_folio(folio, NR_FILE_PAGES, nr);
                __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr);
unlock:
                xas_unlock_irq(&xas);
        } while (xas_nomem(&xas, gfp));

        if (!xas_error(&xas))
                return 0;

        folio_clear_swapcache(folio);
        folio_ref_sub(folio, nr);
        return xas_error(&xas);
}

/*
 * This must be called only on folios that have
 * been verified to be in the swap cache.
 */
void __delete_from_swap_cache(struct folio *folio,
                        swp_entry_t entry, void *shadow)
{
        struct address_space *address_space = swap_address_space(entry);
        int i;
        long nr = folio_nr_pages(folio);
        pgoff_t idx = swap_cache_index(entry);
        XA_STATE(xas, &address_space->i_pages, idx);

        xas_set_update(&xas, workingset_update_node);

        VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio);
        VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio);

        for (i = 0; i < nr; i++) {
                void *entry = xas_store(&xas, shadow);
                VM_BUG_ON_PAGE(entry != folio, entry);
                xas_next(&xas);
        }
        folio->swap.val = 0;
        folio_clear_swapcache(folio);
        address_space->nrpages -= nr;
        __node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
        __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr);
}

/*
 * This must be called only on folios that have
 * been verified to be in the swap cache and locked.
 * It will never put the folio into the free list,
 * the caller has a reference on the folio.
 */
void delete_from_swap_cache(struct folio *folio)
{
        swp_entry_t entry = folio->swap;
        struct address_space *address_space = swap_address_space(entry);

        xa_lock_irq(&address_space->i_pages);
        __delete_from_swap_cache(folio, entry, NULL);
        xa_unlock_irq(&address_space->i_pages);

        put_swap_folio(folio, entry);
        folio_ref_sub(folio, folio_nr_pages(folio));
}

void clear_shadow_from_swap_cache(int type, unsigned long begin,
                                unsigned long end)
{
        unsigned long curr = begin;
        void *old;

        for (;;) {
                swp_entry_t entry = swp_entry(type, curr);
                unsigned long index = curr & SWAP_ADDRESS_SPACE_MASK;
                struct address_space *address_space = swap_address_space(entry);
                XA_STATE(xas, &address_space->i_pages, index);

                xas_set_update(&xas, workingset_update_node);

                xa_lock_irq(&address_space->i_pages);
                xas_for_each(&xas, old, min(index + (end - curr), SWAP_ADDRESS_SPACE_PAGES)) {
                        if (!xa_is_value(old))
                                continue;
                        xas_store(&xas, NULL);
                }
                xa_unlock_irq(&address_space->i_pages);

                /* search the next swapcache until we meet end */
                curr = ALIGN((curr + 1), SWAP_ADDRESS_SPACE_PAGES);
                if (curr > end)
                        break;
        }
}

/*
 * If we are the only user, then try to free up the swap cache.
 *
 * Its ok to check the swapcache flag without the folio lock
 * here because we are going to recheck again inside
 * folio_free_swap() _with_ the lock.
 *                                         - Marcelo
 */
void free_swap_cache(struct folio *folio)
{
        if (folio_test_swapcache(folio) && !folio_mapped(folio) &&
            folio_trylock(folio)) {
                folio_free_swap(folio);
                folio_unlock(folio);
        }
}

/*
 * Freeing a folio and also freeing any swap cache associated with
 * this folio if it is the last user.
 */
void free_folio_and_swap_cache(struct folio *folio)
{
        free_swap_cache(folio);
        if (!is_huge_zero_folio(folio))
                folio_put(folio);
}

/*
 * Passed an array of pages, drop them all from swapcache and then release
 * them.  They are removed from the LRU and freed if this is their last use.
 */
void free_pages_and_swap_cache(struct encoded_page **pages, int nr)
{
        struct folio_batch folios;
        unsigned int refs[PAGEVEC_SIZE];

        folio_batch_init(&folios);
        for (int i = 0; i < nr; i++) {
                struct folio *folio = page_folio(encoded_page_ptr(pages[i]));

                free_swap_cache(folio);
                refs[folios.nr] = 1;
                if (unlikely(encoded_page_flags(pages[i]) &
                             ENCODED_PAGE_BIT_NR_PAGES_NEXT))
                        refs[folios.nr] = encoded_nr_pages(pages[++i]);

                if (folio_batch_add(&folios, folio) == 0)
                        folios_put_refs(&folios, refs);
        }
        if (folios.nr)
                folios_put_refs(&folios, refs);
}

static inline bool swap_use_vma_readahead(void)
{
        return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap);
}

/*
 * Lookup a swap entry in the swap cache. A found folio will be returned
 * unlocked and with its refcount incremented - we rely on the kernel
 * lock getting page table operations atomic even if we drop the folio
 * lock before returning.
 *
 * Caller must lock the swap device or hold a reference to keep it valid.
 */
struct folio *swap_cache_get_folio(swp_entry_t entry,
                struct vm_area_struct *vma, unsigned long addr)
{
        struct folio *folio;

        folio = filemap_get_folio(swap_address_space(entry), swap_cache_index(entry));
        if (!IS_ERR(folio)) {
                bool vma_ra = swap_use_vma_readahead();
                bool readahead;

                /*
                 * At the moment, we don't support PG_readahead for anon THP
                 * so let's bail out rather than confusing the readahead stat.
                 */
                if (unlikely(folio_test_large(folio)))
                        return folio;

                readahead = folio_test_clear_readahead(folio);
                if (vma && vma_ra) {
                        unsigned long ra_val;
                        int win, hits;

                        ra_val = GET_SWAP_RA_VAL(vma);
                        win = SWAP_RA_WIN(ra_val);
                        hits = SWAP_RA_HITS(ra_val);
                        if (readahead)
                                hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX);
                        atomic_long_set(&vma->swap_readahead_info,
                                        SWAP_RA_VAL(addr, win, hits));
                }

                if (readahead) {
                        count_vm_event(SWAP_RA_HIT);
                        if (!vma || !vma_ra)
                                atomic_inc(&swapin_readahead_hits);
                }
        } else {
                folio = NULL;
        }

        return folio;
}

/**
 * filemap_get_incore_folio - Find and get a folio from the page or swap caches.
 * @mapping: The address_space to search.
 * @index: The page cache index.
 *
 * This differs from filemap_get_folio() in that it will also look for the
 * folio in the swap cache.
 *
 * Return: The found folio or %NULL.
 */
struct folio *filemap_get_incore_folio(struct address_space *mapping,
                pgoff_t index)
{
        swp_entry_t swp;
        struct swap_info_struct *si;
        struct folio *folio = filemap_get_entry(mapping, index);

        if (!folio)
                return ERR_PTR(-ENOENT);
        if (!xa_is_value(folio))
                return folio;
        if (!shmem_mapping(mapping))
                return ERR_PTR(-ENOENT);

        swp = radix_to_swp_entry(folio);
        /* There might be swapin error entries in shmem mapping. */
        if (non_swap_entry(swp))
                return ERR_PTR(-ENOENT);
        /* Prevent swapoff from happening to us */
        si = get_swap_device(swp);
        if (!si)
                return ERR_PTR(-ENOENT);
        index = swap_cache_index(swp);
        folio = filemap_get_folio(swap_address_space(swp), index);
        put_swap_device(si);
        return folio;
}

struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated,
                bool skip_if_exists)
{
        struct swap_info_struct *si = swp_swap_info(entry);
        struct folio *folio;
        struct folio *new_folio = NULL;
        struct folio *result = NULL;
        void *shadow = NULL;

        *new_page_allocated = false;
        for (;;) {
                int err;
                /*
                 * First check the swap cache.  Since this is normally
                 * called after swap_cache_get_folio() failed, re-calling
                 * that would confuse statistics.
                 */
                folio = filemap_get_folio(swap_address_space(entry),
                                          swap_cache_index(entry));
                if (!IS_ERR(folio))
                        goto got_folio;

                /*
                 * Just skip read ahead for unused swap slot.
                 */
                if (!swap_entry_swapped(si, entry))
                        goto put_and_return;

                /*
                 * Get a new folio to read into from swap.  Allocate it now if
                 * new_folio not exist, before marking swap_map SWAP_HAS_CACHE,
                 * when -EEXIST will cause any racers to loop around until we
                 * add it to cache.
                 */
                if (!new_folio) {
                        new_folio = folio_alloc_mpol(gfp_mask, 0, mpol, ilx, numa_node_id());
                        if (!new_folio)
                                goto put_and_return;
                }

                /*
                 * Swap entry may have been freed since our caller observed it.
                 */
                err = swapcache_prepare(entry, 1);
                if (!err)
                        break;
                else if (err != -EEXIST)
                        goto put_and_return;

                /*
                 * Protect against a recursive call to __read_swap_cache_async()
                 * on the same entry waiting forever here because SWAP_HAS_CACHE
                 * is set but the folio is not the swap cache yet. This can
                 * happen today if mem_cgroup_swapin_charge_folio() below
                 * triggers reclaim through zswap, which may call
                 * __read_swap_cache_async() in the writeback path.
                 */
                if (skip_if_exists)
                        goto put_and_return;

                /*
                 * We might race against __delete_from_swap_cache(), and
                 * stumble across a swap_map entry whose SWAP_HAS_CACHE
                 * has not yet been cleared.  Or race against another
                 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE
                 * in swap_map, but not yet added its folio to swap cache.
                 */
                schedule_timeout_uninterruptible(1);
        }

        /*
         * The swap entry is ours to swap in. Prepare the new folio.
         */
        __folio_set_locked(new_folio);
        __folio_set_swapbacked(new_folio);

        if (mem_cgroup_swapin_charge_folio(new_folio, NULL, gfp_mask, entry))
                goto fail_unlock;

        /* May fail (-ENOMEM) if XArray node allocation failed. */
        if (add_to_swap_cache(new_folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow))
                goto fail_unlock;

        memcg1_swapin(entry, 1);

        if (shadow)
                workingset_refault(new_folio, shadow);

        /* Caller will initiate read into locked new_folio */
        folio_add_lru(new_folio);
        *new_page_allocated = true;
        folio = new_folio;
got_folio:
        result = folio;
        goto put_and_return;

fail_unlock:
        put_swap_folio(new_folio, entry);
        folio_unlock(new_folio);
put_and_return:
        if (!(*new_page_allocated) && new_folio)
                folio_put(new_folio);
        return result;
}

/*
 * Locate a page of swap in physical memory, reserving swap cache space
 * and reading the disk if it is not already cached.
 * A failure return means that either the page allocation failed or that
 * the swap entry is no longer in use.
 *
 * get/put_swap_device() aren't needed to call this function, because
 * __read_swap_cache_async() call them and swap_read_folio() holds the
 * swap cache folio lock.
 */
struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,
                struct vm_area_struct *vma, unsigned long addr,
                struct swap_iocb **plug)
{
        struct swap_info_struct *si;
        bool page_allocated;
        struct mempolicy *mpol;
        pgoff_t ilx;
        struct folio *folio;

        si = get_swap_device(entry);
        if (!si)
                return NULL;

        mpol = get_vma_policy(vma, addr, 0, &ilx);
        folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
                                        &page_allocated, false);
        mpol_cond_put(mpol);

        if (page_allocated)
                swap_read_folio(folio, plug);

        put_swap_device(si);
        return folio;
}

static unsigned int __swapin_nr_pages(unsigned long prev_offset,
                                      unsigned long offset,
                                      int hits,
                                      int max_pages,
                                      int prev_win)
{
        unsigned int pages, last_ra;

        /*
         * This heuristic has been found to work well on both sequential and
         * random loads, swapping to hard disk or to SSD: please don't ask
         * what the "+ 2" means, it just happens to work well, that's all.
         */
        pages = hits + 2;
        if (pages == 2) {
                /*
                 * We can have no readahead hits to judge by: but must not get
                 * stuck here forever, so check for an adjacent offset instead
                 * (and don't even bother to check whether swap type is same).
                 */
                if (offset != prev_offset + 1 && offset != prev_offset - 1)
                        pages = 1;
        } else {
                unsigned int roundup = 4;
                while (roundup < pages)
                        roundup <<= 1;
                pages = roundup;
        }

        if (pages > max_pages)
                pages = max_pages;

        /* Don't shrink readahead too fast */
        last_ra = prev_win / 2;
        if (pages < last_ra)
                pages = last_ra;

        return pages;
}

static unsigned long swapin_nr_pages(unsigned long offset)
{
        static unsigned long prev_offset;
        unsigned int hits, pages, max_pages;
        static atomic_t last_readahead_pages;

        max_pages = 1 << READ_ONCE(page_cluster);
        if (max_pages <= 1)
                return 1;

        hits = atomic_xchg(&swapin_readahead_hits, 0);
        pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits,
                                  max_pages,
                                  atomic_read(&last_readahead_pages));
        if (!hits)
                WRITE_ONCE(prev_offset, offset);
        atomic_set(&last_readahead_pages, pages);

        return pages;
}

/**
 * swap_cluster_readahead - swap in pages in hope we need them soon
 * @entry: swap entry of this memory
 * @gfp_mask: memory allocation flags
 * @mpol: NUMA memory allocation policy to be applied
 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE
 *
 * Returns the struct folio for entry and addr, after queueing swapin.
 *
 * Primitive swap readahead code. We simply read an aligned block of
 * (1 << page_cluster) entries in the swap area. This method is chosen
 * because it doesn't cost us any seek time.  We also make sure to queue
 * the 'original' request together with the readahead ones...
 *
 * Note: it is intentional that the same NUMA policy and interleave index
 * are used for every page of the readahead: neighbouring pages on swap
 * are fairly likely to have been swapped out from the same node.
 */
struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask,
                                    struct mempolicy *mpol, pgoff_t ilx)
{
        struct folio *folio;
        unsigned long entry_offset = swp_offset(entry);
        unsigned long offset = entry_offset;
        unsigned long start_offset, end_offset;
        unsigned long mask;
        struct swap_info_struct *si = swp_swap_info(entry);
        struct blk_plug plug;
        struct swap_iocb *splug = NULL;
        bool page_allocated;

        mask = swapin_nr_pages(offset) - 1;
        if (!mask)
                goto skip;

        /* Read a page_cluster sized and aligned cluster around offset. */
        start_offset = offset & ~mask;
        end_offset = offset | mask;
        if (!start_offset)        /* First page is swap header. */
                start_offset++;
        if (end_offset >= si->max)
                end_offset = si->max - 1;

        blk_start_plug(&plug);
        for (offset = start_offset; offset <= end_offset ; offset++) {
                /* Ok, do the async read-ahead now */
                folio = __read_swap_cache_async(
                                swp_entry(swp_type(entry), offset),
                                gfp_mask, mpol, ilx, &page_allocated, false);
                if (!folio)
                        continue;
                if (page_allocated) {
                        swap_read_folio(folio, &splug);
                        if (offset != entry_offset) {
                                folio_set_readahead(folio);
                                count_vm_event(SWAP_RA);
                        }
                }
                folio_put(folio);
        }
        blk_finish_plug(&plug);
        swap_read_unplug(splug);
        lru_add_drain();        /* Push any new pages onto the LRU now */
skip:
        /* The page was likely read above, so no need for plugging here */
        folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
                                        &page_allocated, false);
        if (unlikely(page_allocated))
                swap_read_folio(folio, NULL);
        return folio;
}

int init_swap_address_space(unsigned int type, unsigned long nr_pages)
{
        struct address_space *spaces, *space;
        unsigned int i, nr;

        nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
        spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL);
        if (!spaces)
                return -ENOMEM;
        for (i = 0; i < nr; i++) {
                space = spaces + i;
                xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ);
                atomic_set(&space->i_mmap_writable, 0);
                space->a_ops = &swap_aops;
                /* swap cache doesn't use writeback related tags */
                mapping_set_no_writeback_tags(space);
        }
        nr_swapper_spaces[type] = nr;
        swapper_spaces[type] = spaces;

        return 0;
}

void exit_swap_address_space(unsigned int type)
{
        int i;
        struct address_space *spaces = swapper_spaces[type];

        for (i = 0; i < nr_swapper_spaces[type]; i++)
                VM_WARN_ON_ONCE(!mapping_empty(&spaces[i]));
        kvfree(spaces);
        nr_swapper_spaces[type] = 0;
        swapper_spaces[type] = NULL;
}

static int swap_vma_ra_win(struct vm_fault *vmf, unsigned long *start,
                           unsigned long *end)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long ra_val;
        unsigned long faddr, prev_faddr, left, right;
        unsigned int max_win, hits, prev_win, win;

        max_win = 1 << min(READ_ONCE(page_cluster), SWAP_RA_ORDER_CEILING);
        if (max_win == 1)
                return 1;

        faddr = vmf->address;
        ra_val = GET_SWAP_RA_VAL(vma);
        prev_faddr = SWAP_RA_ADDR(ra_val);
        prev_win = SWAP_RA_WIN(ra_val);
        hits = SWAP_RA_HITS(ra_val);
        win = __swapin_nr_pages(PFN_DOWN(prev_faddr), PFN_DOWN(faddr), hits,
                                max_win, prev_win);
        atomic_long_set(&vma->swap_readahead_info, SWAP_RA_VAL(faddr, win, 0));
        if (win == 1)
                return 1;

        if (faddr == prev_faddr + PAGE_SIZE)
                left = faddr;
        else if (prev_faddr == faddr + PAGE_SIZE)
                left = faddr - (win << PAGE_SHIFT) + PAGE_SIZE;
        else
                left = faddr - (((win - 1) / 2) << PAGE_SHIFT);
        right = left + (win << PAGE_SHIFT);
        if ((long)left < 0)
                left = 0;
        *start = max3(left, vma->vm_start, faddr & PMD_MASK);
        *end = min3(right, vma->vm_end, (faddr & PMD_MASK) + PMD_SIZE);

        return win;
}

/**
 * swap_vma_readahead - swap in pages in hope we need them soon
 * @targ_entry: swap entry of the targeted memory
 * @gfp_mask: memory allocation flags
 * @mpol: NUMA memory allocation policy to be applied
 * @targ_ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE
 * @vmf: fault information
 *
 * Returns the struct folio for entry and addr, after queueing swapin.
 *
 * Primitive swap readahead code. We simply read in a few pages whose
 * virtual addresses are around the fault address in the same vma.
 *
 * Caller must hold read mmap_lock if vmf->vma is not NULL.
 *
 */
static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask,
                struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf)
{
        struct blk_plug plug;
        struct swap_iocb *splug = NULL;
        struct folio *folio;
        pte_t *pte = NULL, pentry;
        int win;
        unsigned long start, end, addr;
        swp_entry_t entry;
        pgoff_t ilx;
        bool page_allocated;

        win = swap_vma_ra_win(vmf, &start, &end);
        if (win == 1)
                goto skip;

        ilx = targ_ilx - PFN_DOWN(vmf->address - start);

        blk_start_plug(&plug);
        for (addr = start; addr < end; ilx++, addr += PAGE_SIZE) {
                if (!pte++) {
                        pte = pte_offset_map(vmf->pmd, addr);
                        if (!pte)
                                break;
                }
                pentry = ptep_get_lockless(pte);
                if (!is_swap_pte(pentry))
                        continue;
                entry = pte_to_swp_entry(pentry);
                if (unlikely(non_swap_entry(entry)))
                        continue;
                pte_unmap(pte);
                pte = NULL;
                folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx,
                                                &page_allocated, false);
                if (!folio)
                        continue;
                if (page_allocated) {
                        swap_read_folio(folio, &splug);
                        if (addr != vmf->address) {
                                folio_set_readahead(folio);
                                count_vm_event(SWAP_RA);
                        }
                }
                folio_put(folio);
        }
        if (pte)
                pte_unmap(pte);
        blk_finish_plug(&plug);
        swap_read_unplug(splug);
        lru_add_drain();
skip:
        /* The folio was likely read above, so no need for plugging here */
        folio = __read_swap_cache_async(targ_entry, gfp_mask, mpol, targ_ilx,
                                        &page_allocated, false);
        if (unlikely(page_allocated))
                swap_read_folio(folio, NULL);
        return folio;
}

/**
 * swapin_readahead - swap in pages in hope we need them soon
 * @entry: swap entry of this memory
 * @gfp_mask: memory allocation flags
 * @vmf: fault information
 *
 * Returns the struct folio for entry and addr, after queueing swapin.
 *
 * It's a main entry function for swap readahead. By the configuration,
 * it will read ahead blocks by cluster-based(ie, physical disk based)
 * or vma-based(ie, virtual address based on faulty address) readahead.
 */
struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
                                struct vm_fault *vmf)
{
        struct mempolicy *mpol;
        pgoff_t ilx;
        struct folio *folio;

        mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx);
        folio = swap_use_vma_readahead() ?
                swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) :
                swap_cluster_readahead(entry, gfp_mask, mpol, ilx);
        mpol_cond_put(mpol);

        return folio;
}

#ifdef CONFIG_SYSFS
static ssize_t vma_ra_enabled_show(struct kobject *kobj,
                                     struct kobj_attribute *attr, char *buf)
{
        return sysfs_emit(buf, "%s\n", str_true_false(enable_vma_readahead));
}
static ssize_t vma_ra_enabled_store(struct kobject *kobj,
                                      struct kobj_attribute *attr,
                                      const char *buf, size_t count)
{
        ssize_t ret;

        ret = kstrtobool(buf, &enable_vma_readahead);
        if (ret)
                return ret;

        return count;
}
static struct kobj_attribute vma_ra_enabled_attr = __ATTR_RW(vma_ra_enabled);

static struct attribute *swap_attrs[] = {
        &vma_ra_enabled_attr.attr,
        NULL,
};

static const struct attribute_group swap_attr_group = {
        .attrs = swap_attrs,
};

static int __init swap_init_sysfs(void)
{
        int err;
        struct kobject *swap_kobj;

        swap_kobj = kobject_create_and_add("swap", mm_kobj);
        if (!swap_kobj) {
                pr_err("failed to create swap kobject\n");
                return -ENOMEM;
        }
        err = sysfs_create_group(swap_kobj, &swap_attr_group);
        if (err) {
                pr_err("failed to register swap group\n");
                goto delete_obj;
        }
        return 0;

delete_obj:
        kobject_put(swap_kobj);
        return err;
}
subsys_initcall(swap_init_sysfs);
#endif





































































































































































































































































































































































































































































































































  171 
  172 



















































































































































































































































































































































































































































  172 












  172 


































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Implementation of the policy database.
 *
 * Author : Stephen Smalley, <stephen.smalley.work@gmail.com>
 */

/*
 * Updated: Trusted Computer Solutions, Inc. <dgoeddel@trustedcs.com>
 *          Support for enhanced MLS infrastructure.
 *          Copyright (C) 2004-2005 Trusted Computer Solutions, Inc.
 *
 * Updated: Frank Mayer <mayerf@tresys.com> and
 *          Karl MacMillan <kmacmillan@tresys.com>
 *          Added conditional policy language extensions
 *          Copyright (C) 2003-2004 Tresys Technology, LLC
 *
 * Updated: Hewlett-Packard <paul@paul-moore.com>
 *          Added support for the policy capability bitmap
 *          Copyright (C) 2007 Hewlett-Packard Development Company, L.P.
 *
 * Update: Mellanox Techonologies
 *         Added Infiniband support
 *         Copyright (C) 2016 Mellanox Techonologies
 */

#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/audit.h>
#include "security.h"

#include "policydb.h"
#include "conditional.h"
#include "mls.h"
#include "services.h"

#ifdef CONFIG_SECURITY_SELINUX_DEBUG
/* clang-format off */
static const char *const symtab_name[SYM_NUM] = {
        "common prefixes",
        "classes",
        "roles",
        "types",
        "users",
        "bools",
        "levels",
        "categories",
};
/* clang-format off */
#endif

struct policydb_compat_info {
        unsigned int version;
        unsigned int sym_num;
        unsigned int ocon_num;
};

/* These need to be updated if SYM_NUM or OCON_NUM changes */
static const struct policydb_compat_info policydb_compat[] = {
        {
                .version = POLICYDB_VERSION_BASE,
                .sym_num = SYM_NUM - 3,
                .ocon_num = OCON_NUM - 3,
        },
        {
                .version = POLICYDB_VERSION_BOOL,
                .sym_num = SYM_NUM - 2,
                .ocon_num = OCON_NUM - 3,
        },
        {
                .version = POLICYDB_VERSION_IPV6,
                .sym_num = SYM_NUM - 2,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_NLCLASS,
                .sym_num = SYM_NUM - 2,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_MLS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_AVTAB,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_RANGETRANS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_POLCAP,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_PERMISSIVE,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_BOUNDARY,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_FILENAME_TRANS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_ROLETRANS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_NEW_OBJECT_DEFAULTS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_DEFAULT_TYPE,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_CONSTRAINT_NAMES,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_XPERMS_IOCTL,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM - 2,
        },
        {
                .version = POLICYDB_VERSION_INFINIBAND,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM,
        },
        {
                .version = POLICYDB_VERSION_GLBLUB,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM,
        },
        {
                .version = POLICYDB_VERSION_COMP_FTRANS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM,
        },
        {
                .version = POLICYDB_VERSION_COND_XPERMS,
                .sym_num = SYM_NUM,
                .ocon_num = OCON_NUM,
        },
};

static const struct policydb_compat_info *
policydb_lookup_compat(unsigned int version)
{
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(policydb_compat); i++) {
                if (policydb_compat[i].version == version)
                        return &policydb_compat[i];
        }

        return NULL;
}

/*
 * The following *_destroy functions are used to
 * free any memory allocated for each kind of
 * symbol data in the policy database.
 */

static int perm_destroy(void *key, void *datum, void *p)
{
        kfree(key);
        kfree(datum);
        return 0;
}

static int common_destroy(void *key, void *datum, void *p)
{
        struct common_datum *comdatum;

        kfree(key);
        if (datum) {
                comdatum = datum;
                hashtab_map(&comdatum->permissions.table, perm_destroy, NULL);
                hashtab_destroy(&comdatum->permissions.table);
        }
        kfree(datum);
        return 0;
}

static void constraint_expr_destroy(struct constraint_expr *expr)
{
        if (expr) {
                ebitmap_destroy(&expr->names);
                if (expr->type_names) {
                        ebitmap_destroy(&expr->type_names->types);
                        ebitmap_destroy(&expr->type_names->negset);
                        kfree(expr->type_names);
                }
                kfree(expr);
        }
}

static int cls_destroy(void *key, void *datum, void *p)
{
        struct class_datum *cladatum;
        struct constraint_node *constraint, *ctemp;
        struct constraint_expr *e, *etmp;

        kfree(key);
        if (datum) {
                cladatum = datum;
                hashtab_map(&cladatum->permissions.table, perm_destroy, NULL);
                hashtab_destroy(&cladatum->permissions.table);
                constraint = cladatum->constraints;
                while (constraint) {
                        e = constraint->expr;
                        while (e) {
                                etmp = e;
                                e = e->next;
                                constraint_expr_destroy(etmp);
                        }
                        ctemp = constraint;
                        constraint = constraint->next;
                        kfree(ctemp);
                }

                constraint = cladatum->validatetrans;
                while (constraint) {
                        e = constraint->expr;
                        while (e) {
                                etmp = e;
                                e = e->next;
                                constraint_expr_destroy(etmp);
                        }
                        ctemp = constraint;
                        constraint = constraint->next;
                        kfree(ctemp);
                }
                kfree(cladatum->comkey);
        }
        kfree(datum);
        return 0;
}

static int role_destroy(void *key, void *datum, void *p)
{
        struct role_datum *role;

        kfree(key);
        if (datum) {
                role = datum;
                ebitmap_destroy(&role->dominates);
                ebitmap_destroy(&role->types);
        }
        kfree(datum);
        return 0;
}

static int type_destroy(void *key, void *datum, void *p)
{
        kfree(key);
        kfree(datum);
        return 0;
}

static int user_destroy(void *key, void *datum, void *p)
{
        struct user_datum *usrdatum;

        kfree(key);
        if (datum) {
                usrdatum = datum;
                ebitmap_destroy(&usrdatum->roles);
                ebitmap_destroy(&usrdatum->range.level[0].cat);
                ebitmap_destroy(&usrdatum->range.level[1].cat);
                ebitmap_destroy(&usrdatum->dfltlevel.cat);
        }
        kfree(datum);
        return 0;
}

static int sens_destroy(void *key, void *datum, void *p)
{
        struct level_datum *levdatum;

        kfree(key);
        if (datum) {
                levdatum = datum;
                ebitmap_destroy(&levdatum->level.cat);
        }
        kfree(datum);
        return 0;
}

static int cat_destroy(void *key, void *datum, void *p)
{
        kfree(key);
        kfree(datum);
        return 0;
}

/* clang-format off */
static int (*const destroy_f[SYM_NUM])(void *key, void *datum, void *datap) = {
        common_destroy,
        cls_destroy,
        role_destroy,
        type_destroy,
        user_destroy,
        cond_destroy_bool,
        sens_destroy,
        cat_destroy,
};
/* clang-format on */

static int filenametr_destroy(void *key, void *datum, void *p)
{
        struct filename_trans_key *ft = key;
        struct filename_trans_datum *next, *d = datum;

        kfree(ft->name);
        kfree(key);
        do {
                ebitmap_destroy(&d->stypes);
                next = d->next;
                kfree(d);
                d = next;
        } while (unlikely(d));
        cond_resched();
        return 0;
}

static int range_tr_destroy(void *key, void *datum, void *p)
{
        struct mls_range *rt = datum;

        kfree(key);
        ebitmap_destroy(&rt->level[0].cat);
        ebitmap_destroy(&rt->level[1].cat);
        kfree(datum);
        cond_resched();
        return 0;
}

static int role_tr_destroy(void *key, void *datum, void *p)
{
        kfree(key);
        kfree(datum);
        return 0;
}

static void ocontext_destroy(struct ocontext *c, unsigned int i)
{
        if (!c)
                return;

        context_destroy(&c->context[0]);
        context_destroy(&c->context[1]);
        if (i == OCON_ISID || i == OCON_FS || i == OCON_NETIF ||
            i == OCON_FSUSE)
                kfree(c->u.name);
        kfree(c);
}

/*
 * Initialize the role table.
 */
static int roles_init(struct policydb *p)
{
        char *key = NULL;
        int rc;
        struct role_datum *role;

        role = kzalloc(sizeof(*role), GFP_KERNEL);
        if (!role)
                return -ENOMEM;

        rc = -EINVAL;
        role->value = ++p->p_roles.nprim;
        if (role->value != OBJECT_R_VAL)
                goto out;

        rc = -ENOMEM;
        key = kstrdup(OBJECT_R, GFP_KERNEL);
        if (!key)
                goto out;

        rc = symtab_insert(&p->p_roles, key, role);
        if (rc)
                goto out;

        return 0;
out:
        kfree(key);
        kfree(role);
        return rc;
}

static u32 filenametr_hash(const void *k)
{
        const struct filename_trans_key *ft = k;
        unsigned long salt = ft->ttype ^ ft->tclass;

        return full_name_hash((void *)salt, ft->name, strlen(ft->name));
}

static int filenametr_cmp(const void *k1, const void *k2)
{
        const struct filename_trans_key *ft1 = k1;
        const struct filename_trans_key *ft2 = k2;
        int v;

        v = ft1->ttype - ft2->ttype;
        if (v)
                return v;

        v = ft1->tclass - ft2->tclass;
        if (v)
                return v;

        return strcmp(ft1->name, ft2->name);
}

static const struct hashtab_key_params filenametr_key_params = {
        .hash = filenametr_hash,
        .cmp = filenametr_cmp,
};

struct filename_trans_datum *
policydb_filenametr_search(struct policydb *p, struct filename_trans_key *key)
{
        return hashtab_search(&p->filename_trans, key, filenametr_key_params);
}

static u32 rangetr_hash(const void *k)
{
        const struct range_trans *key = k;

        return key->source_type + (key->target_type << 3) +
               (key->target_class << 5);
}

static int rangetr_cmp(const void *k1, const void *k2)
{
        const struct range_trans *key1 = k1, *key2 = k2;
        int v;

        v = key1->source_type - key2->source_type;
        if (v)
                return v;

        v = key1->target_type - key2->target_type;
        if (v)
                return v;

        v = key1->target_class - key2->target_class;

        return v;
}

static const struct hashtab_key_params rangetr_key_params = {
        .hash = rangetr_hash,
        .cmp = rangetr_cmp,
};

struct mls_range *policydb_rangetr_search(struct policydb *p,
                                          struct range_trans *key)
{
        return hashtab_search(&p->range_tr, key, rangetr_key_params);
}

static u32 role_trans_hash(const void *k)
{
        const struct role_trans_key *key = k;

        return jhash_3words(key->role, key->type,
                            (u32)key->tclass << 16 | key->tclass, 0);
}

static int role_trans_cmp(const void *k1, const void *k2)
{
        const struct role_trans_key *key1 = k1, *key2 = k2;
        int v;

        v = key1->role - key2->role;
        if (v)
                return v;

        v = key1->type - key2->type;
        if (v)
                return v;

        return key1->tclass - key2->tclass;
}

static const struct hashtab_key_params roletr_key_params = {
        .hash = role_trans_hash,
        .cmp = role_trans_cmp,
};

struct role_trans_datum *policydb_roletr_search(struct policydb *p,
                                                struct role_trans_key *key)
{
        return hashtab_search(&p->role_tr, key, roletr_key_params);
}

/*
 * Initialize a policy database structure.
 */
static void policydb_init(struct policydb *p)
{
        memset(p, 0, sizeof(*p));

        avtab_init(&p->te_avtab);
        cond_policydb_init(p);

        ebitmap_init(&p->filename_trans_ttypes);
        ebitmap_init(&p->policycaps);
        ebitmap_init(&p->permissive_map);
}

/*
 * The following *_index functions are used to
 * define the val_to_name and val_to_struct arrays
 * in a policy database structure.  The val_to_name
 * arrays are used when converting security context
 * structures into string representations.  The
 * val_to_struct arrays are used when the attributes
 * of a class, role, or user are needed.
 */

static int common_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct common_datum *comdatum;

        comdatum = datum;
        p = datap;
        if (!comdatum->value || comdatum->value > p->p_commons.nprim)
                return -EINVAL;

        p->sym_val_to_name[SYM_COMMONS][comdatum->value - 1] = key;

        return 0;
}

static int class_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct class_datum *cladatum;

        cladatum = datum;
        p = datap;
        if (!cladatum->value || cladatum->value > p->p_classes.nprim)
                return -EINVAL;

        p->sym_val_to_name[SYM_CLASSES][cladatum->value - 1] = key;
        p->class_val_to_struct[cladatum->value - 1] = cladatum;
        return 0;
}

static int role_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct role_datum *role;

        role = datum;
        p = datap;
        if (!role->value || role->value > p->p_roles.nprim ||
            role->bounds > p->p_roles.nprim)
                return -EINVAL;

        p->sym_val_to_name[SYM_ROLES][role->value - 1] = key;
        p->role_val_to_struct[role->value - 1] = role;
        return 0;
}

static int type_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct type_datum *typdatum;

        typdatum = datum;
        p = datap;

        if (typdatum->primary) {
                if (!typdatum->value || typdatum->value > p->p_types.nprim ||
                    typdatum->bounds > p->p_types.nprim)
                        return -EINVAL;
                p->sym_val_to_name[SYM_TYPES][typdatum->value - 1] = key;
                p->type_val_to_struct[typdatum->value - 1] = typdatum;
        }

        return 0;
}

static int user_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct user_datum *usrdatum;

        usrdatum = datum;
        p = datap;
        if (!usrdatum->value || usrdatum->value > p->p_users.nprim ||
            usrdatum->bounds > p->p_users.nprim)
                return -EINVAL;

        p->sym_val_to_name[SYM_USERS][usrdatum->value - 1] = key;
        p->user_val_to_struct[usrdatum->value - 1] = usrdatum;
        return 0;
}

static int sens_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct level_datum *levdatum;

        levdatum = datum;
        p = datap;

        if (!levdatum->isalias) {
                if (!levdatum->level.sens ||
                    levdatum->level.sens > p->p_levels.nprim)
                        return -EINVAL;

                p->sym_val_to_name[SYM_LEVELS][levdatum->level.sens - 1] = key;
        }

        return 0;
}

static int cat_index(void *key, void *datum, void *datap)
{
        struct policydb *p;
        struct cat_datum *catdatum;

        catdatum = datum;
        p = datap;

        if (!catdatum->isalias) {
                if (!catdatum->value || catdatum->value > p->p_cats.nprim)
                        return -EINVAL;

                p->sym_val_to_name[SYM_CATS][catdatum->value - 1] = key;
        }

        return 0;
}

/* clang-format off */
static int (*const index_f[SYM_NUM])(void *key, void *datum, void *datap) = {
        common_index,
        class_index,
        role_index,
        type_index,
        user_index,
        cond_index_bool,
        sens_index,
        cat_index,
};
/* clang-format on */

#ifdef CONFIG_SECURITY_SELINUX_DEBUG
static void hash_eval(struct hashtab *h, const char *hash_name,
                      const char *hash_details)
{
        struct hashtab_info info;

        hashtab_stat(h, &info);
        pr_debug(
                "SELinux: %s%s%s:  %d entries and %d/%d buckets used, longest chain length %d, sum of chain length^2 %llu\n",
                hash_name, hash_details ? "@" : "", hash_details ?: "", h->nel,
                info.slots_used, h->size, info.max_chain_len,
                info.chain2_len_sum);
}

static void symtab_hash_eval(struct symtab *s)
{
        int i;

        for (i = 0; i < SYM_NUM; i++)
                hash_eval(&s[i].table, symtab_name[i], NULL);
}

#else
static inline void hash_eval(struct hashtab *h, const char *hash_name,
                             const char *hash_details)
{
}
static inline void symtab_hash_eval(struct symtab *s)
{
}
#endif /* CONFIG_SECURITY_SELINUX_DEBUG */

/*
 * Define the other val_to_name and val_to_struct arrays
 * in a policy database structure.
 *
 * Caller must clean up on failure.
 */
static int policydb_index(struct policydb *p)
{
        int i, rc;

        if (p->mls_enabled)
                pr_debug(
                        "SELinux:  %d users, %d roles, %d types, %d bools, %d sens, %d cats\n",
                        p->p_users.nprim, p->p_roles.nprim, p->p_types.nprim,
                        p->p_bools.nprim, p->p_levels.nprim, p->p_cats.nprim);
        else
                pr_debug("SELinux:  %d users, %d roles, %d types, %d bools\n",
                         p->p_users.nprim, p->p_roles.nprim, p->p_types.nprim,
                         p->p_bools.nprim);

        pr_debug("SELinux:  %d classes, %d rules\n", p->p_classes.nprim,
                 p->te_avtab.nel);

        avtab_hash_eval(&p->te_avtab, "rules");
        symtab_hash_eval(p->symtab);

        p->class_val_to_struct = kcalloc(p->p_classes.nprim,
                                         sizeof(*p->class_val_to_struct),
                                         GFP_KERNEL);
        if (!p->class_val_to_struct)
                return -ENOMEM;

        p->role_val_to_struct = kcalloc(
                p->p_roles.nprim, sizeof(*p->role_val_to_struct), GFP_KERNEL);
        if (!p->role_val_to_struct)
                return -ENOMEM;

        p->user_val_to_struct = kcalloc(
                p->p_users.nprim, sizeof(*p->user_val_to_struct), GFP_KERNEL);
        if (!p->user_val_to_struct)
                return -ENOMEM;

        p->type_val_to_struct = kvcalloc(
                p->p_types.nprim, sizeof(*p->type_val_to_struct), GFP_KERNEL);
        if (!p->type_val_to_struct)
                return -ENOMEM;

        rc = cond_init_bool_indexes(p);
        if (rc)
                goto out;

        for (i = 0; i < SYM_NUM; i++) {
                p->sym_val_to_name[i] = kvcalloc(p->symtab[i].nprim,
                                                 sizeof(char *), GFP_KERNEL);
                if (!p->sym_val_to_name[i])
                        return -ENOMEM;

                rc = hashtab_map(&p->symtab[i].table, index_f[i], p);
                if (rc)
                        goto out;
        }
        rc = 0;
out:
        return rc;
}

/*
 * Free any memory allocated by a policy database structure.
 */
void policydb_destroy(struct policydb *p)
{
        struct ocontext *c, *ctmp;
        struct genfs *g, *gtmp;
        u32 i;
        struct role_allow *ra, *lra = NULL;

        for (i = 0; i < SYM_NUM; i++) {
                cond_resched();
                hashtab_map(&p->symtab[i].table, destroy_f[i], NULL);
                hashtab_destroy(&p->symtab[i].table);
        }

        for (i = 0; i < SYM_NUM; i++)
                kvfree(p->sym_val_to_name[i]);

        kfree(p->class_val_to_struct);
        kfree(p->role_val_to_struct);
        kfree(p->user_val_to_struct);
        kvfree(p->type_val_to_struct);

        avtab_destroy(&p->te_avtab);

        for (i = 0; i < OCON_NUM; i++) {
                cond_resched();
                c = p->ocontexts[i];
                while (c) {
                        ctmp = c;
                        c = c->next;
                        ocontext_destroy(ctmp, i);
                }
                p->ocontexts[i] = NULL;
        }

        g = p->genfs;
        while (g) {
                cond_resched();
                kfree(g->fstype);
                c = g->head;
                while (c) {
                        ctmp = c;
                        c = c->next;
                        ocontext_destroy(ctmp, OCON_FSUSE);
                }
                gtmp = g;
                g = g->next;
                kfree(gtmp);
        }
        p->genfs = NULL;

        cond_policydb_destroy(p);

        hashtab_map(&p->role_tr, role_tr_destroy, NULL);
        hashtab_destroy(&p->role_tr);

        for (ra = p->role_allow; ra; ra = ra->next) {
                cond_resched();
                kfree(lra);
                lra = ra;
        }
        kfree(lra);

        hashtab_map(&p->filename_trans, filenametr_destroy, NULL);
        hashtab_destroy(&p->filename_trans);

        hashtab_map(&p->range_tr, range_tr_destroy, NULL);
        hashtab_destroy(&p->range_tr);

        if (p->type_attr_map_array) {
                for (i = 0; i < p->p_types.nprim; i++)
                        ebitmap_destroy(&p->type_attr_map_array[i]);
                kvfree(p->type_attr_map_array);
        }

        ebitmap_destroy(&p->filename_trans_ttypes);
        ebitmap_destroy(&p->policycaps);
        ebitmap_destroy(&p->permissive_map);
}

/*
 * Load the initial SIDs specified in a policy database
 * structure into a SID table.
 */
int policydb_load_isids(struct policydb *p, struct sidtab *s)
{
        struct ocontext *head, *c;
        bool isid_init;
        int rc;

        rc = sidtab_init(s);
        if (rc) {
                pr_err("SELinux:  out of memory on SID table init\n");
                return rc;
        }

        isid_init = ebitmap_get_bit(&p->policycaps,
                                    POLICYDB_CAP_USERSPACE_INITIAL_CONTEXT);

        head = p->ocontexts[OCON_ISID];
        for (c = head; c; c = c->next) {
                u32 sid = c->sid[0];
                const char *name = security_get_initial_sid_context(sid);

                if (sid == SECSID_NULL) {
                        pr_err("SELinux:  SID 0 was assigned a context.\n");
                        sidtab_destroy(s);
                        return -EINVAL;
                }

                /* Ignore initial SIDs unused by this kernel. */
                if (!name)
                        continue;

                /*
                 * Also ignore SECINITSID_INIT if the policy doesn't declare
                 * support for it
                 */
                if (sid == SECINITSID_INIT && !isid_init)
                        continue;

                rc = sidtab_set_initial(s, sid, &c->context[0]);
                if (rc) {
                        pr_err("SELinux:  unable to load initial SID %s.\n",
                               name);
                        sidtab_destroy(s);
                        return rc;
                }

                /*
                 * If the policy doesn't support the "userspace_initial_context"
                 * capability, set SECINITSID_INIT to the same context as
                 * SECINITSID_KERNEL. This ensures the same behavior as before
                 * the reintroduction of SECINITSID_INIT, where all tasks
                 * started before policy load would initially get the context
                 * corresponding to SECINITSID_KERNEL.
                 */
                if (sid == SECINITSID_KERNEL && !isid_init) {
                        rc = sidtab_set_initial(s, SECINITSID_INIT,
                                                &c->context[0]);
                        if (rc) {
                                pr_err("SELinux:  unable to load initial SID %s.\n",
                                       name);
                                sidtab_destroy(s);
                                return rc;
                        }
                }
        }
        return 0;
}

int policydb_class_isvalid(struct policydb *p, unsigned int class)
{
        if (!class || class > p->p_classes.nprim)
                return 0;
        return 1;
}

int policydb_role_isvalid(struct policydb *p, unsigned int role)
{
        if (!role || role > p->p_roles.nprim)
                return 0;
        return 1;
}

int policydb_type_isvalid(struct policydb *p, unsigned int type)
{
        if (!type || type > p->p_types.nprim)
                return 0;
        return 1;
}

/*
 * Return 1 if the fields in the security context
 * structure `c' are valid.  Return 0 otherwise.
 */
int policydb_context_isvalid(struct policydb *p, struct context *c)
{
        struct role_datum *role;
        struct user_datum *usrdatum;

        if (!c->role || c->role > p->p_roles.nprim)
                return 0;

        if (!c->user || c->user > p->p_users.nprim)
                return 0;

        if (!c->type || c->type > p->p_types.nprim)
                return 0;

        if (c->role != OBJECT_R_VAL) {
                /*
                 * Role must be authorized for the type.
                 */
                role = p->role_val_to_struct[c->role - 1];
                if (!role || !ebitmap_get_bit(&role->types, c->type - 1))
                        /* role may not be associated with type */
                        return 0;

                /*
                 * User must be authorized for the role.
                 */
                usrdatum = p->user_val_to_struct[c->user - 1];
                if (!usrdatum)
                        return 0;

                if (!ebitmap_get_bit(&usrdatum->roles, c->role - 1))
                        /* user may not be associated with role */
                        return 0;
        }

        if (!mls_context_isvalid(p, c))
                return 0;

        return 1;
}

/*
 * Read a MLS range structure from a policydb binary
 * representation file.
 */
static int mls_read_range_helper(struct mls_range *r, struct policy_file *fp)
{
        __le32 buf[2];
        u32 items;
        int rc;

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                goto out;

        rc = -EINVAL;
        items = le32_to_cpu(buf[0]);
        if (items > ARRAY_SIZE(buf)) {
                pr_err("SELinux: mls:  range overflow\n");
                goto out;
        }

        rc = next_entry(buf, fp, sizeof(u32) * items);
        if (rc) {
                pr_err("SELinux: mls:  truncated range\n");
                goto out;
        }

        r->level[0].sens = le32_to_cpu(buf[0]);
        if (items > 1)
                r->level[1].sens = le32_to_cpu(buf[1]);
        else
                r->level[1].sens = r->level[0].sens;

        rc = ebitmap_read(&r->level[0].cat, fp);
        if (rc) {
                pr_err("SELinux: mls:  error reading low categories\n");
                goto out;
        }
        if (items > 1) {
                rc = ebitmap_read(&r->level[1].cat, fp);
                if (rc) {
                        pr_err("SELinux: mls:  error reading high categories\n");
                        goto bad_high;
                }
        } else {
                rc = ebitmap_cpy(&r->level[1].cat, &r->level[0].cat);
                if (rc) {
                        pr_err("SELinux: mls:  out of memory\n");
                        goto bad_high;
                }
        }

        return 0;
bad_high:
        ebitmap_destroy(&r->level[0].cat);
out:
        return rc;
}

/*
 * Read and validate a security context structure
 * from a policydb binary representation file.
 */
static int context_read_and_validate(struct context *c, struct policydb *p,
                                     struct policy_file *fp)
{
        __le32 buf[3];
        int rc;

        rc = next_entry(buf, fp, sizeof buf);
        if (rc) {
                pr_err("SELinux: context truncated\n");
                goto out;
        }
        c->user = le32_to_cpu(buf[0]);
        c->role = le32_to_cpu(buf[1]);
        c->type = le32_to_cpu(buf[2]);
        if (p->policyvers >= POLICYDB_VERSION_MLS) {
                rc = mls_read_range_helper(&c->range, fp);
                if (rc) {
                        pr_err("SELinux: error reading MLS range of context\n");
                        goto out;
                }
        }

        rc = -EINVAL;
        if (!policydb_context_isvalid(p, c)) {
                pr_err("SELinux:  invalid security context\n");
                context_destroy(c);
                goto out;
        }
        rc = 0;
out:
        return rc;
}

/*
 * The following *_read functions are used to
 * read the symbol data from a policy database
 * binary representation file.
 */

int str_read(char **strp, gfp_t flags, struct policy_file *fp, u32 len)
{
        int rc;
        char *str;

        if ((len == 0) || (len == (u32)-1))
                return -EINVAL;

        str = kmalloc(len + 1, flags | __GFP_NOWARN);
        if (!str)
                return -ENOMEM;

        rc = next_entry(str, fp, len);
        if (rc) {
                kfree(str);
                return rc;
        }

        str[len] = '\0';
        *strp = str;
        return 0;
}

static int perm_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct perm_datum *perdatum;
        int rc;
        __le32 buf[2];
        u32 len;

        perdatum = kzalloc(sizeof(*perdatum), GFP_KERNEL);
        if (!perdatum)
                return -ENOMEM;

        rc = next_entry(buf, fp, sizeof buf);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        perdatum->value = le32_to_cpu(buf[1]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = symtab_insert(s, key, perdatum);
        if (rc)
                goto bad;

        return 0;
bad:
        perm_destroy(key, perdatum, NULL);
        return rc;
}

static int common_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct common_datum *comdatum;
        __le32 buf[4];
        u32 i, len, nel;
        int rc;

        comdatum = kzalloc(sizeof(*comdatum), GFP_KERNEL);
        if (!comdatum)
                return -ENOMEM;

        rc = next_entry(buf, fp, sizeof buf);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        comdatum->value = le32_to_cpu(buf[1]);
        nel = le32_to_cpu(buf[3]);

        rc = symtab_init(&comdatum->permissions, nel);
        if (rc)
                goto bad;
        comdatum->permissions.nprim = le32_to_cpu(buf[2]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        for (i = 0; i < nel; i++) {
                rc = perm_read(p, &comdatum->permissions, fp);
                if (rc)
                        goto bad;
        }

        hash_eval(&comdatum->permissions.table, "common_permissions", key);

        rc = symtab_insert(s, key, comdatum);
        if (rc)
                goto bad;
        return 0;
bad:
        common_destroy(key, comdatum, NULL);
        return rc;
}

static void type_set_init(struct type_set *t)
{
        ebitmap_init(&t->types);
        ebitmap_init(&t->negset);
}

static int type_set_read(struct type_set *t, struct policy_file *fp)
{
        __le32 buf[1];
        int rc;

        if (ebitmap_read(&t->types, fp))
                return -EINVAL;
        if (ebitmap_read(&t->negset, fp))
                return -EINVAL;

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc < 0)
                return -EINVAL;
        t->flags = le32_to_cpu(buf[0]);

        return 0;
}

static int read_cons_helper(struct policydb *p, struct constraint_node **nodep,
                            u32 ncons, int allowxtarget, struct policy_file *fp)
{
        struct constraint_node *c, *lc;
        struct constraint_expr *e, *le;
        __le32 buf[3];
        u32 i, j, nexpr;
        int rc, depth;

        lc = NULL;
        for (i = 0; i < ncons; i++) {
                c = kzalloc(sizeof(*c), GFP_KERNEL);
                if (!c)
                        return -ENOMEM;

                if (lc)
                        lc->next = c;
                else
                        *nodep = c;

                rc = next_entry(buf, fp, (sizeof(u32) * 2));
                if (rc)
                        return rc;
                c->permissions = le32_to_cpu(buf[0]);
                nexpr = le32_to_cpu(buf[1]);
                le = NULL;
                depth = -1;
                for (j = 0; j < nexpr; j++) {
                        e = kzalloc(sizeof(*e), GFP_KERNEL);
                        if (!e)
                                return -ENOMEM;

                        if (le)
                                le->next = e;
                        else
                                c->expr = e;

                        rc = next_entry(buf, fp, (sizeof(u32) * 3));
                        if (rc)
                                return rc;
                        e->expr_type = le32_to_cpu(buf[0]);
                        e->attr = le32_to_cpu(buf[1]);
                        e->op = le32_to_cpu(buf[2]);

                        switch (e->expr_type) {
                        case CEXPR_NOT:
                                if (depth < 0)
                                        return -EINVAL;
                                break;
                        case CEXPR_AND:
                        case CEXPR_OR:
                                if (depth < 1)
                                        return -EINVAL;
                                depth--;
                                break;
                        case CEXPR_ATTR:
                                if (depth == (CEXPR_MAXDEPTH - 1))
                                        return -EINVAL;
                                depth++;
                                break;
                        case CEXPR_NAMES:
                                if (!allowxtarget && (e->attr & CEXPR_XTARGET))
                                        return -EINVAL;
                                if (depth == (CEXPR_MAXDEPTH - 1))
                                        return -EINVAL;
                                depth++;
                                rc = ebitmap_read(&e->names, fp);
                                if (rc)
                                        return rc;
                                if (p->policyvers >=
                                    POLICYDB_VERSION_CONSTRAINT_NAMES) {
                                        e->type_names =
                                                kzalloc(sizeof(*e->type_names),
                                                        GFP_KERNEL);
                                        if (!e->type_names)
                                                return -ENOMEM;
                                        type_set_init(e->type_names);
                                        rc = type_set_read(e->type_names, fp);
                                        if (rc)
                                                return rc;
                                }
                                break;
                        default:
                                return -EINVAL;
                        }
                        le = e;
                }
                if (depth != 0)
                        return -EINVAL;
                lc = c;
        }

        return 0;
}

static int class_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct class_datum *cladatum;
        __le32 buf[6];
        u32 i, len, len2, ncons, nel;
        int rc;

        cladatum = kzalloc(sizeof(*cladatum), GFP_KERNEL);
        if (!cladatum)
                return -ENOMEM;

        rc = next_entry(buf, fp, sizeof(u32) * 6);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        len2 = le32_to_cpu(buf[1]);
        cladatum->value = le32_to_cpu(buf[2]);
        nel = le32_to_cpu(buf[4]);

        rc = symtab_init(&cladatum->permissions, nel);
        if (rc)
                goto bad;
        cladatum->permissions.nprim = le32_to_cpu(buf[3]);

        ncons = le32_to_cpu(buf[5]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        if (len2) {
                rc = str_read(&cladatum->comkey, GFP_KERNEL, fp, len2);
                if (rc)
                        goto bad;

                rc = -EINVAL;
                cladatum->comdatum =
                        symtab_search(&p->p_commons, cladatum->comkey);
                if (!cladatum->comdatum) {
                        pr_err("SELinux:  unknown common %s\n",
                               cladatum->comkey);
                        goto bad;
                }
        }
        for (i = 0; i < nel; i++) {
                rc = perm_read(p, &cladatum->permissions, fp);
                if (rc)
                        goto bad;
        }

        hash_eval(&cladatum->permissions.table, "class_permissions", key);

        rc = read_cons_helper(p, &cladatum->constraints, ncons, 0, fp);
        if (rc)
                goto bad;

        if (p->policyvers >= POLICYDB_VERSION_VALIDATETRANS) {
                /* grab the validatetrans rules */
                rc = next_entry(buf, fp, sizeof(u32));
                if (rc)
                        goto bad;
                ncons = le32_to_cpu(buf[0]);
                rc = read_cons_helper(p, &cladatum->validatetrans, ncons, 1,
                                      fp);
                if (rc)
                        goto bad;
        }

        if (p->policyvers >= POLICYDB_VERSION_NEW_OBJECT_DEFAULTS) {
                rc = next_entry(buf, fp, sizeof(u32) * 3);
                if (rc)
                        goto bad;

                cladatum->default_user = le32_to_cpu(buf[0]);
                cladatum->default_role = le32_to_cpu(buf[1]);
                cladatum->default_range = le32_to_cpu(buf[2]);
        }

        if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) {
                rc = next_entry(buf, fp, sizeof(u32) * 1);
                if (rc)
                        goto bad;
                cladatum->default_type = le32_to_cpu(buf[0]);
        }

        rc = symtab_insert(s, key, cladatum);
        if (rc)
                goto bad;

        return 0;
bad:
        cls_destroy(key, cladatum, NULL);
        return rc;
}

static int role_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct role_datum *role;
        int rc;
        unsigned int to_read = 2;
        __le32 buf[3];
        u32 len;

        role = kzalloc(sizeof(*role), GFP_KERNEL);
        if (!role)
                return -ENOMEM;

        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                to_read = 3;

        rc = next_entry(buf, fp, sizeof(buf[0]) * to_read);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        role->value = le32_to_cpu(buf[1]);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                role->bounds = le32_to_cpu(buf[2]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = ebitmap_read(&role->dominates, fp);
        if (rc)
                goto bad;

        rc = ebitmap_read(&role->types, fp);
        if (rc)
                goto bad;

        if (strcmp(key, OBJECT_R) == 0) {
                rc = -EINVAL;
                if (role->value != OBJECT_R_VAL) {
                        pr_err("SELinux: Role %s has wrong value %d\n",
                               OBJECT_R, role->value);
                        goto bad;
                }
                rc = 0;
                goto bad;
        }

        rc = symtab_insert(s, key, role);
        if (rc)
                goto bad;
        return 0;
bad:
        role_destroy(key, role, NULL);
        return rc;
}

static int type_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct type_datum *typdatum;
        int rc;
        unsigned int to_read = 3;
        __le32 buf[4];
        u32 len;

        typdatum = kzalloc(sizeof(*typdatum), GFP_KERNEL);
        if (!typdatum)
                return -ENOMEM;

        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                to_read = 4;

        rc = next_entry(buf, fp, sizeof(buf[0]) * to_read);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        typdatum->value = le32_to_cpu(buf[1]);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY) {
                u32 prop = le32_to_cpu(buf[2]);

                if (prop & TYPEDATUM_PROPERTY_PRIMARY)
                        typdatum->primary = 1;
                if (prop & TYPEDATUM_PROPERTY_ATTRIBUTE)
                        typdatum->attribute = 1;

                typdatum->bounds = le32_to_cpu(buf[3]);
        } else {
                typdatum->primary = le32_to_cpu(buf[2]);
        }

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = symtab_insert(s, key, typdatum);
        if (rc)
                goto bad;
        return 0;
bad:
        type_destroy(key, typdatum, NULL);
        return rc;
}

/*
 * Read a MLS level structure from a policydb binary
 * representation file.
 */
static int mls_read_level(struct mls_level *lp, struct policy_file *fp)
{
        __le32 buf[1];
        int rc;

        memset(lp, 0, sizeof(*lp));

        rc = next_entry(buf, fp, sizeof buf);
        if (rc) {
                pr_err("SELinux: mls: truncated level\n");
                return rc;
        }
        lp->sens = le32_to_cpu(buf[0]);

        rc = ebitmap_read(&lp->cat, fp);
        if (rc) {
                pr_err("SELinux: mls:  error reading level categories\n");
                return rc;
        }
        return 0;
}

static int user_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct user_datum *usrdatum;
        int rc;
        unsigned int to_read = 2;
        __le32 buf[3];
        u32 len;

        usrdatum = kzalloc(sizeof(*usrdatum), GFP_KERNEL);
        if (!usrdatum)
                return -ENOMEM;

        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                to_read = 3;

        rc = next_entry(buf, fp, sizeof(buf[0]) * to_read);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        usrdatum->value = le32_to_cpu(buf[1]);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                usrdatum->bounds = le32_to_cpu(buf[2]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = ebitmap_read(&usrdatum->roles, fp);
        if (rc)
                goto bad;

        if (p->policyvers >= POLICYDB_VERSION_MLS) {
                rc = mls_read_range_helper(&usrdatum->range, fp);
                if (rc)
                        goto bad;
                rc = mls_read_level(&usrdatum->dfltlevel, fp);
                if (rc)
                        goto bad;
        }

        rc = symtab_insert(s, key, usrdatum);
        if (rc)
                goto bad;
        return 0;
bad:
        user_destroy(key, usrdatum, NULL);
        return rc;
}

static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct level_datum *levdatum;
        int rc;
        __le32 buf[2];
        u32 len;

        levdatum = kzalloc(sizeof(*levdatum), GFP_KERNEL);
        if (!levdatum)
                return -ENOMEM;

        rc = next_entry(buf, fp, sizeof buf);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        levdatum->isalias = le32_to_cpu(buf[1]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = mls_read_level(&levdatum->level, fp);
        if (rc)
                goto bad;

        rc = symtab_insert(s, key, levdatum);
        if (rc)
                goto bad;
        return 0;
bad:
        sens_destroy(key, levdatum, NULL);
        return rc;
}

static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp)
{
        char *key = NULL;
        struct cat_datum *catdatum;
        int rc;
        __le32 buf[3];
        u32 len;

        catdatum = kzalloc(sizeof(*catdatum), GFP_KERNEL);
        if (!catdatum)
                return -ENOMEM;

        rc = next_entry(buf, fp, sizeof buf);
        if (rc)
                goto bad;

        len = le32_to_cpu(buf[0]);
        catdatum->value = le32_to_cpu(buf[1]);
        catdatum->isalias = le32_to_cpu(buf[2]);

        rc = str_read(&key, GFP_KERNEL, fp, len);
        if (rc)
                goto bad;

        rc = symtab_insert(s, key, catdatum);
        if (rc)
                goto bad;
        return 0;
bad:
        cat_destroy(key, catdatum, NULL);
        return rc;
}

/* clang-format off */
static int (*const read_f[SYM_NUM])(struct policydb *p, struct symtab *s,
                                    struct policy_file *fp) = {
        common_read,
        class_read,
        role_read,
        type_read,
        user_read,
        cond_read_bool,
        sens_read,
        cat_read,
};
/* clang-format on */

static int user_bounds_sanity_check(void *key, void *datum, void *datap)
{
        struct user_datum *upper, *user;
        struct policydb *p = datap;
        int depth = 0;

        upper = user = datum;
        while (upper->bounds) {
                struct ebitmap_node *node;
                u32 bit;

                if (++depth == POLICYDB_BOUNDS_MAXDEPTH) {
                        pr_err("SELinux: user %s: "
                               "too deep or looped boundary\n",
                               (char *)key);
                        return -EINVAL;
                }

                upper = p->user_val_to_struct[upper->bounds - 1];
                ebitmap_for_each_positive_bit(&user->roles, node, bit)
                {
                        if (ebitmap_get_bit(&upper->roles, bit))
                                continue;

                        pr_err("SELinux: boundary violated policy: "
                               "user=%s role=%s bounds=%s\n",
                               sym_name(p, SYM_USERS, user->value - 1),
                               sym_name(p, SYM_ROLES, bit),
                               sym_name(p, SYM_USERS, upper->value - 1));

                        return -EINVAL;
                }
        }

        return 0;
}

static int role_bounds_sanity_check(void *key, void *datum, void *datap)
{
        struct role_datum *upper, *role;
        struct policydb *p = datap;
        int depth = 0;

        upper = role = datum;
        while (upper->bounds) {
                struct ebitmap_node *node;
                u32 bit;

                if (++depth == POLICYDB_BOUNDS_MAXDEPTH) {
                        pr_err("SELinux: role %s: "
                               "too deep or looped bounds\n",
                               (char *)key);
                        return -EINVAL;
                }

                upper = p->role_val_to_struct[upper->bounds - 1];
                ebitmap_for_each_positive_bit(&role->types, node, bit)
                {
                        if (ebitmap_get_bit(&upper->types, bit))
                                continue;

                        pr_err("SELinux: boundary violated policy: "
                               "role=%s type=%s bounds=%s\n",
                               sym_name(p, SYM_ROLES, role->value - 1),
                               sym_name(p, SYM_TYPES, bit),
                               sym_name(p, SYM_ROLES, upper->value - 1));

                        return -EINVAL;
                }
        }

        return 0;
}

static int type_bounds_sanity_check(void *key, void *datum, void *datap)
{
        struct type_datum *upper;
        struct policydb *p = datap;
        int depth = 0;

        upper = datum;
        while (upper->bounds) {
                if (++depth == POLICYDB_BOUNDS_MAXDEPTH) {
                        pr_err("SELinux: type %s: "
                               "too deep or looped boundary\n",
                               (char *)key);
                        return -EINVAL;
                }

                upper = p->type_val_to_struct[upper->bounds - 1];
                BUG_ON(!upper);

                if (upper->attribute) {
                        pr_err("SELinux: type %s: "
                               "bounded by attribute %s\n",
                               (char *)key,
                               sym_name(p, SYM_TYPES, upper->value - 1));
                        return -EINVAL;
                }
        }

        return 0;
}

static int policydb_bounds_sanity_check(struct policydb *p)
{
        int rc;

        if (p->policyvers < POLICYDB_VERSION_BOUNDARY)
                return 0;

        rc = hashtab_map(&p->p_users.table, user_bounds_sanity_check, p);
        if (rc)
                return rc;

        rc = hashtab_map(&p->p_roles.table, role_bounds_sanity_check, p);
        if (rc)
                return rc;

        rc = hashtab_map(&p->p_types.table, type_bounds_sanity_check, p);
        if (rc)
                return rc;

        return 0;
}

u16 string_to_security_class(struct policydb *p, const char *name)
{
        struct class_datum *cladatum;

        cladatum = symtab_search(&p->p_classes, name);
        if (!cladatum)
                return 0;

        return cladatum->value;
}

u32 string_to_av_perm(struct policydb *p, u16 tclass, const char *name)
{
        struct class_datum *cladatum;
        struct perm_datum *perdatum = NULL;
        struct common_datum *comdatum;

        if (!tclass || tclass > p->p_classes.nprim)
                return 0;

        cladatum = p->class_val_to_struct[tclass - 1];
        comdatum = cladatum->comdatum;
        if (comdatum)
                perdatum = symtab_search(&comdatum->permissions, name);
        if (!perdatum)
                perdatum = symtab_search(&cladatum->permissions, name);
        if (!perdatum)
                return 0;

        return 1U << (perdatum->value - 1);
}

static int range_read(struct policydb *p, struct policy_file *fp)
{
        struct range_trans *rt = NULL;
        struct mls_range *r = NULL;
        int rc;
        __le32 buf[2];
        u32 i, nel;

        if (p->policyvers < POLICYDB_VERSION_MLS)
                return 0;

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                return rc;

        nel = le32_to_cpu(buf[0]);

        rc = hashtab_init(&p->range_tr, nel);
        if (rc)
                return rc;

        for (i = 0; i < nel; i++) {
                rc = -ENOMEM;
                rt = kzalloc(sizeof(*rt), GFP_KERNEL);
                if (!rt)
                        goto out;

                rc = next_entry(buf, fp, (sizeof(u32) * 2));
                if (rc)
                        goto out;

                rt->source_type = le32_to_cpu(buf[0]);
                rt->target_type = le32_to_cpu(buf[1]);
                if (p->policyvers >= POLICYDB_VERSION_RANGETRANS) {
                        rc = next_entry(buf, fp, sizeof(u32));
                        if (rc)
                                goto out;
                        rt->target_class = le32_to_cpu(buf[0]);
                } else
                        rt->target_class = p->process_class;

                rc = -EINVAL;
                if (!policydb_type_isvalid(p, rt->source_type) ||
                    !policydb_type_isvalid(p, rt->target_type) ||
                    !policydb_class_isvalid(p, rt->target_class))
                        goto out;

                rc = -ENOMEM;
                r = kzalloc(sizeof(*r), GFP_KERNEL);
                if (!r)
                        goto out;

                rc = mls_read_range_helper(r, fp);
                if (rc)
                        goto out;

                rc = -EINVAL;
                if (!mls_range_isvalid(p, r)) {
                        pr_warn("SELinux:  rangetrans:  invalid range\n");
                        goto out;
                }

                rc = hashtab_insert(&p->range_tr, rt, r, rangetr_key_params);
                if (rc)
                        goto out;

                rt = NULL;
                r = NULL;
        }
        hash_eval(&p->range_tr, "rangetr", NULL);
        rc = 0;
out:
        kfree(rt);
        kfree(r);
        return rc;
}

static int filename_trans_read_helper_compat(struct policydb *p, struct policy_file *fp)
{
        struct filename_trans_key key, *ft = NULL;
        struct filename_trans_datum *last, *datum = NULL;
        char *name = NULL;
        u32 len, stype, otype;
        __le32 buf[4];
        int rc;

        /* length of the path component string */
        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                return rc;
        len = le32_to_cpu(buf[0]);

        /* path component string */
        rc = str_read(&name, GFP_KERNEL, fp, len);
        if (rc)
                return rc;

        rc = next_entry(buf, fp, sizeof(u32) * 4);
        if (rc)
                goto out;

        stype = le32_to_cpu(buf[0]);
        key.ttype = le32_to_cpu(buf[1]);
        key.tclass = le32_to_cpu(buf[2]);
        key.name = name;

        otype = le32_to_cpu(buf[3]);

        last = NULL;
        datum = policydb_filenametr_search(p, &key);
        while (datum) {
                if (unlikely(ebitmap_get_bit(&datum->stypes, stype - 1))) {
                        /* conflicting/duplicate rules are ignored */
                        datum = NULL;
                        rc = 0;
                        goto out;
                }
                if (likely(datum->otype == otype))
                        break;
                last = datum;
                datum = datum->next;
        }
        if (!datum) {
                rc = -ENOMEM;
                datum = kmalloc(sizeof(*datum), GFP_KERNEL);
                if (!datum)
                        goto out;

                ebitmap_init(&datum->stypes);
                datum->otype = otype;
                datum->next = NULL;

                if (unlikely(last)) {
                        last->next = datum;
                } else {
                        rc = -ENOMEM;
                        ft = kmemdup(&key, sizeof(key), GFP_KERNEL);
                        if (!ft)
                                goto out;

                        rc = hashtab_insert(&p->filename_trans, ft, datum,
                                            filenametr_key_params);
                        if (rc)
                                goto out;
                        name = NULL;

                        rc = ebitmap_set_bit(&p->filename_trans_ttypes,
                                             key.ttype, 1);
                        if (rc)
                                return rc;
                }
        }
        kfree(name);
        return ebitmap_set_bit(&datum->stypes, stype - 1, 1);

out:
        kfree(ft);
        kfree(name);
        kfree(datum);
        return rc;
}

static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp)
{
        struct filename_trans_key *ft = NULL;
        struct filename_trans_datum **dst, *datum, *first = NULL;
        char *name = NULL;
        u32 len, ttype, tclass, ndatum, i;
        __le32 buf[3];
        int rc;

        /* length of the path component string */
        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                return rc;
        len = le32_to_cpu(buf[0]);

        /* path component string */
        rc = str_read(&name, GFP_KERNEL, fp, len);
        if (rc)
                return rc;

        rc = next_entry(buf, fp, sizeof(u32) * 3);
        if (rc)
                goto out;

        ttype = le32_to_cpu(buf[0]);
        tclass = le32_to_cpu(buf[1]);

        ndatum = le32_to_cpu(buf[2]);
        if (ndatum == 0) {
                pr_err("SELinux:  Filename transition key with no datum\n");
                rc = -ENOENT;
                goto out;
        }

        dst = &first;
        for (i = 0; i < ndatum; i++) {
                rc = -ENOMEM;
                datum = kmalloc(sizeof(*datum), GFP_KERNEL);
                if (!datum)
                        goto out;

                datum->next = NULL;
                *dst = datum;

                /* ebitmap_read() will at least init the bitmap */
                rc = ebitmap_read(&datum->stypes, fp);
                if (rc)
                        goto out;

                rc = next_entry(buf, fp, sizeof(u32));
                if (rc)
                        goto out;

                datum->otype = le32_to_cpu(buf[0]);

                dst = &datum->next;
        }

        rc = -ENOMEM;
        ft = kmalloc(sizeof(*ft), GFP_KERNEL);
        if (!ft)
                goto out;

        ft->ttype = ttype;
        ft->tclass = tclass;
        ft->name = name;

        rc = hashtab_insert(&p->filename_trans, ft, first,
                            filenametr_key_params);
        if (rc == -EEXIST)
                pr_err("SELinux:  Duplicate filename transition key\n");
        if (rc)
                goto out;

        return ebitmap_set_bit(&p->filename_trans_ttypes, ttype, 1);

out:
        kfree(ft);
        kfree(name);
        while (first) {
                datum = first;
                first = first->next;

                ebitmap_destroy(&datum->stypes);
                kfree(datum);
        }
        return rc;
}

static int filename_trans_read(struct policydb *p, struct policy_file *fp)
{
        u32 nel, i;
        __le32 buf[1];
        int rc;

        if (p->policyvers < POLICYDB_VERSION_FILENAME_TRANS)
                return 0;

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                return rc;
        nel = le32_to_cpu(buf[0]);

        if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) {
                p->compat_filename_trans_count = nel;

                rc = hashtab_init(&p->filename_trans, (1 << 11));
                if (rc)
                        return rc;

                for (i = 0; i < nel; i++) {
                        rc = filename_trans_read_helper_compat(p, fp);
                        if (rc)
                                return rc;
                }
        } else {
                rc = hashtab_init(&p->filename_trans, nel);
                if (rc)
                        return rc;

                for (i = 0; i < nel; i++) {
                        rc = filename_trans_read_helper(p, fp);
                        if (rc)
                                return rc;
                }
        }
        hash_eval(&p->filename_trans, "filenametr", NULL);
        return 0;
}

static int genfs_read(struct policydb *p, struct policy_file *fp)
{
        int rc;
        u32 i, j, nel, nel2, len, len2;
        __le32 buf[1];
        struct ocontext *l, *c;
        struct ocontext *newc = NULL;
        struct genfs *genfs_p, *genfs;
        struct genfs *newgenfs = NULL;

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                return rc;
        nel = le32_to_cpu(buf[0]);

        for (i = 0; i < nel; i++) {
                rc = next_entry(buf, fp, sizeof(u32));
                if (rc)
                        goto out;
                len = le32_to_cpu(buf[0]);

                rc = -ENOMEM;
                newgenfs = kzalloc(sizeof(*newgenfs), GFP_KERNEL);
                if (!newgenfs)
                        goto out;

                rc = str_read(&newgenfs->fstype, GFP_KERNEL, fp, len);
                if (rc)
                        goto out;

                for (genfs_p = NULL, genfs = p->genfs; genfs;
                     genfs_p = genfs, genfs = genfs->next) {
                        rc = -EINVAL;
                        if (strcmp(newgenfs->fstype, genfs->fstype) == 0) {
                                pr_err("SELinux:  dup genfs fstype %s\n",
                                       newgenfs->fstype);
                                goto out;
                        }
                        if (strcmp(newgenfs->fstype, genfs->fstype) < 0)
                                break;
                }
                newgenfs->next = genfs;
                if (genfs_p)
                        genfs_p->next = newgenfs;
                else
                        p->genfs = newgenfs;
                genfs = newgenfs;
                newgenfs = NULL;

                rc = next_entry(buf, fp, sizeof(u32));
                if (rc)
                        goto out;

                nel2 = le32_to_cpu(buf[0]);
                for (j = 0; j < nel2; j++) {
                        rc = next_entry(buf, fp, sizeof(u32));
                        if (rc)
                                goto out;
                        len = le32_to_cpu(buf[0]);

                        rc = -ENOMEM;
                        newc = kzalloc(sizeof(*newc), GFP_KERNEL);
                        if (!newc)
                                goto out;

                        rc = str_read(&newc->u.name, GFP_KERNEL, fp, len);
                        if (rc)
                                goto out;

                        rc = next_entry(buf, fp, sizeof(u32));
                        if (rc)
                                goto out;

                        newc->v.sclass = le32_to_cpu(buf[0]);
                        rc = context_read_and_validate(&newc->context[0], p,
                                                       fp);
                        if (rc)
                                goto out;

                        for (l = NULL, c = genfs->head; c; l = c, c = c->next) {
                                rc = -EINVAL;
                                if (!strcmp(newc->u.name, c->u.name) &&
                                    (!c->v.sclass || !newc->v.sclass ||
                                     newc->v.sclass == c->v.sclass)) {
                                        pr_err("SELinux:  dup genfs entry (%s,%s)\n",
                                               genfs->fstype, c->u.name);
                                        goto out;
                                }
                                len = strlen(newc->u.name);
                                len2 = strlen(c->u.name);
                                if (len > len2)
                                        break;
                        }

                        newc->next = c;
                        if (l)
                                l->next = newc;
                        else
                                genfs->head = newc;
                        newc = NULL;
                }
        }
        rc = 0;
out:
        if (newgenfs) {
                kfree(newgenfs->fstype);
                kfree(newgenfs);
        }
        ocontext_destroy(newc, OCON_FSUSE);

        return rc;
}

static int ocontext_read(struct policydb *p,
                         const struct policydb_compat_info *info, struct policy_file *fp)
{
        int rc;
        unsigned int i;
        u32 j, nel, len;
        __be64 prefixbuf[1];
        __le32 buf[3];
        struct ocontext *l, *c;
        u32 nodebuf[8];

        for (i = 0; i < info->ocon_num; i++) {
                rc = next_entry(buf, fp, sizeof(u32));
                if (rc)
                        goto out;
                nel = le32_to_cpu(buf[0]);

                l = NULL;
                for (j = 0; j < nel; j++) {
                        rc = -ENOMEM;
                        c = kzalloc(sizeof(*c), GFP_KERNEL);
                        if (!c)
                                goto out;
                        if (l)
                                l->next = c;
                        else
                                p->ocontexts[i] = c;
                        l = c;

                        switch (i) {
                        case OCON_ISID:
                                rc = next_entry(buf, fp, sizeof(u32));
                                if (rc)
                                        goto out;

                                c->sid[0] = le32_to_cpu(buf[0]);
                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        case OCON_FS:
                        case OCON_NETIF:
                                rc = next_entry(buf, fp, sizeof(u32));
                                if (rc)
                                        goto out;
                                len = le32_to_cpu(buf[0]);

                                rc = str_read(&c->u.name, GFP_KERNEL, fp, len);
                                if (rc)
                                        goto out;

                                if (i == OCON_FS)
                                        pr_warn("SELinux:  void and deprecated fs ocon %s\n",
                                                c->u.name);

                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                rc = context_read_and_validate(&c->context[1],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        case OCON_PORT:
                                rc = next_entry(buf, fp, sizeof(u32) * 3);
                                if (rc)
                                        goto out;
                                c->u.port.protocol = le32_to_cpu(buf[0]);
                                c->u.port.low_port = le32_to_cpu(buf[1]);
                                c->u.port.high_port = le32_to_cpu(buf[2]);
                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        case OCON_NODE:
                                rc = next_entry(nodebuf, fp, sizeof(u32) * 2);
                                if (rc)
                                        goto out;
                                c->u.node.addr = nodebuf[0]; /* network order */
                                c->u.node.mask = nodebuf[1]; /* network order */
                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        case OCON_FSUSE:
                                rc = next_entry(buf, fp, sizeof(u32) * 2);
                                if (rc)
                                        goto out;

                                rc = -EINVAL;
                                c->v.behavior = le32_to_cpu(buf[0]);
                                /* Determined at runtime, not in policy DB. */
                                if (c->v.behavior == SECURITY_FS_USE_MNTPOINT)
                                        goto out;
                                if (c->v.behavior > SECURITY_FS_USE_MAX)
                                        goto out;

                                len = le32_to_cpu(buf[1]);
                                rc = str_read(&c->u.name, GFP_KERNEL, fp, len);
                                if (rc)
                                        goto out;

                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        case OCON_NODE6: {
                                int k;

                                rc = next_entry(nodebuf, fp, sizeof(u32) * 8);
                                if (rc)
                                        goto out;
                                for (k = 0; k < 4; k++)
                                        c->u.node6.addr[k] = nodebuf[k];
                                for (k = 0; k < 4; k++)
                                        c->u.node6.mask[k] = nodebuf[k + 4];
                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        }
                        case OCON_IBPKEY: {
                                u32 pkey_lo, pkey_hi;

                                rc = next_entry(prefixbuf, fp, sizeof(u64));
                                if (rc)
                                        goto out;

                                /* we need to have subnet_prefix in CPU order */
                                c->u.ibpkey.subnet_prefix =
                                        be64_to_cpu(prefixbuf[0]);

                                rc = next_entry(buf, fp, sizeof(u32) * 2);
                                if (rc)
                                        goto out;

                                pkey_lo = le32_to_cpu(buf[0]);
                                pkey_hi = le32_to_cpu(buf[1]);

                                if (pkey_lo > U16_MAX || pkey_hi > U16_MAX) {
                                        rc = -EINVAL;
                                        goto out;
                                }

                                c->u.ibpkey.low_pkey = pkey_lo;
                                c->u.ibpkey.high_pkey = pkey_hi;

                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        }
                        case OCON_IBENDPORT: {
                                u32 port;

                                rc = next_entry(buf, fp, sizeof(u32) * 2);
                                if (rc)
                                        goto out;
                                len = le32_to_cpu(buf[0]);

                                rc = str_read(&c->u.ibendport.dev_name,
                                              GFP_KERNEL, fp, len);
                                if (rc)
                                        goto out;

                                port = le32_to_cpu(buf[1]);
                                if (port > U8_MAX || port == 0) {
                                        rc = -EINVAL;
                                        goto out;
                                }

                                c->u.ibendport.port = port;

                                rc = context_read_and_validate(&c->context[0],
                                                               p, fp);
                                if (rc)
                                        goto out;
                                break;
                        } /* end case */
                        } /* end switch */
                }
        }
        rc = 0;
out:
        return rc;
}

/*
 * Read the configuration data from a policy database binary
 * representation file into a policy database structure.
 */
int policydb_read(struct policydb *p, struct policy_file *fp)
{
        struct role_allow *ra, *lra;
        struct role_trans_key *rtk = NULL;
        struct role_trans_datum *rtd = NULL;
        int rc;
        __le32 buf[4];
        u32 i, j, len, nprim, nel, perm;

        char *policydb_str;
        const struct policydb_compat_info *info;

        policydb_init(p);

        /* Read the magic number and string length. */
        rc = next_entry(buf, fp, sizeof(u32) * 2);
        if (rc)
                goto bad;

        rc = -EINVAL;
        if (le32_to_cpu(buf[0]) != POLICYDB_MAGIC) {
                pr_err("SELinux:  policydb magic number 0x%x does "
                       "not match expected magic number 0x%x\n",
                       le32_to_cpu(buf[0]), POLICYDB_MAGIC);
                goto bad;
        }

        rc = -EINVAL;
        len = le32_to_cpu(buf[1]);
        if (len != strlen(POLICYDB_STRING)) {
                pr_err("SELinux:  policydb string length %d does not "
                       "match expected length %zu\n",
                       len, strlen(POLICYDB_STRING));
                goto bad;
        }

        rc = str_read(&policydb_str, GFP_KERNEL, fp, len);
        if (rc) {
                if (rc == -ENOMEM) {
                        pr_err("SELinux:  unable to allocate memory for policydb string of length %d\n",
                               len);
                } else {
                        pr_err("SELinux:  truncated policydb string identifier\n");
                }
                goto bad;
        }

        rc = -EINVAL;
        if (strcmp(policydb_str, POLICYDB_STRING)) {
                pr_err("SELinux:  policydb string %s does not match "
                       "my string %s\n",
                       policydb_str, POLICYDB_STRING);
                kfree(policydb_str);
                goto bad;
        }
        /* Done with policydb_str. */
        kfree(policydb_str);
        policydb_str = NULL;

        /* Read the version and table sizes. */
        rc = next_entry(buf, fp, sizeof(u32) * 4);
        if (rc)
                goto bad;

        rc = -EINVAL;
        p->policyvers = le32_to_cpu(buf[0]);
        if (p->policyvers < POLICYDB_VERSION_MIN ||
            p->policyvers > POLICYDB_VERSION_MAX) {
                pr_err("SELinux:  policydb version %d does not match "
                       "my version range %d-%d\n",
                       le32_to_cpu(buf[0]), POLICYDB_VERSION_MIN,
                       POLICYDB_VERSION_MAX);
                goto bad;
        }

        if ((le32_to_cpu(buf[1]) & POLICYDB_CONFIG_MLS)) {
                p->mls_enabled = 1;

                rc = -EINVAL;
                if (p->policyvers < POLICYDB_VERSION_MLS) {
                        pr_err("SELinux: security policydb version %d "
                               "(MLS) not backwards compatible\n",
                               p->policyvers);
                        goto bad;
                }
        }
        p->reject_unknown = !!(le32_to_cpu(buf[1]) & REJECT_UNKNOWN);
        p->allow_unknown = !!(le32_to_cpu(buf[1]) & ALLOW_UNKNOWN);

        if (p->policyvers >= POLICYDB_VERSION_POLCAP) {
                rc = ebitmap_read(&p->policycaps, fp);
                if (rc)
                        goto bad;
        }

        if (p->policyvers >= POLICYDB_VERSION_PERMISSIVE) {
                rc = ebitmap_read(&p->permissive_map, fp);
                if (rc)
                        goto bad;
        }

        rc = -EINVAL;
        info = policydb_lookup_compat(p->policyvers);
        if (!info) {
                pr_err("SELinux:  unable to find policy compat info "
                       "for version %d\n",
                       p->policyvers);
                goto bad;
        }

        rc = -EINVAL;
        if (le32_to_cpu(buf[2]) != info->sym_num ||
            le32_to_cpu(buf[3]) != info->ocon_num) {
                pr_err("SELinux:  policydb table sizes (%d,%d) do "
                       "not match mine (%d,%d)\n",
                       le32_to_cpu(buf[2]), le32_to_cpu(buf[3]), info->sym_num,
                       info->ocon_num);
                goto bad;
        }

        for (i = 0; i < info->sym_num; i++) {
                rc = next_entry(buf, fp, sizeof(u32) * 2);
                if (rc)
                        goto bad;
                nprim = le32_to_cpu(buf[0]);
                nel = le32_to_cpu(buf[1]);

                rc = symtab_init(&p->symtab[i], nel);
                if (rc)
                        goto out;

                if (i == SYM_ROLES) {
                        rc = roles_init(p);
                        if (rc)
                                goto out;
                }

                for (j = 0; j < nel; j++) {
                        rc = read_f[i](p, &p->symtab[i], fp);
                        if (rc)
                                goto bad;
                }

                p->symtab[i].nprim = nprim;
        }

        rc = -EINVAL;
        p->process_class = string_to_security_class(p, "process");
        if (!p->process_class) {
                pr_err("SELinux: process class is required, not defined in policy\n");
                goto bad;
        }

        rc = avtab_read(&p->te_avtab, fp, p);
        if (rc)
                goto bad;

        if (p->policyvers >= POLICYDB_VERSION_BOOL) {
                rc = cond_read_list(p, fp);
                if (rc)
                        goto bad;
        }

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                goto bad;
        nel = le32_to_cpu(buf[0]);

        rc = hashtab_init(&p->role_tr, nel);
        if (rc)
                goto bad;
        for (i = 0; i < nel; i++) {
                rc = -ENOMEM;
                rtk = kmalloc(sizeof(*rtk), GFP_KERNEL);
                if (!rtk)
                        goto bad;

                rc = -ENOMEM;
                rtd = kmalloc(sizeof(*rtd), GFP_KERNEL);
                if (!rtd)
                        goto bad;

                rc = next_entry(buf, fp, sizeof(u32) * 3);
                if (rc)
                        goto bad;

                rtk->role = le32_to_cpu(buf[0]);
                rtk->type = le32_to_cpu(buf[1]);
                rtd->new_role = le32_to_cpu(buf[2]);
                if (p->policyvers >= POLICYDB_VERSION_ROLETRANS) {
                        rc = next_entry(buf, fp, sizeof(u32));
                        if (rc)
                                goto bad;
                        rtk->tclass = le32_to_cpu(buf[0]);
                } else
                        rtk->tclass = p->process_class;

                rc = -EINVAL;
                if (!policydb_role_isvalid(p, rtk->role) ||
                    !policydb_type_isvalid(p, rtk->type) ||
                    !policydb_class_isvalid(p, rtk->tclass) ||
                    !policydb_role_isvalid(p, rtd->new_role))
                        goto bad;

                rc = hashtab_insert(&p->role_tr, rtk, rtd, roletr_key_params);
                if (rc)
                        goto bad;

                rtk = NULL;
                rtd = NULL;
        }

        hash_eval(&p->role_tr, "roletr", NULL);

        rc = next_entry(buf, fp, sizeof(u32));
        if (rc)
                goto bad;
        nel = le32_to_cpu(buf[0]);
        lra = NULL;
        for (i = 0; i < nel; i++) {
                rc = -ENOMEM;
                ra = kzalloc(sizeof(*ra), GFP_KERNEL);
                if (!ra)
                        goto bad;
                if (lra)
                        lra->next = ra;
                else
                        p->role_allow = ra;
                rc = next_entry(buf, fp, sizeof(u32) * 2);
                if (rc)
                        goto bad;

                rc = -EINVAL;
                ra->role = le32_to_cpu(buf[0]);
                ra->new_role = le32_to_cpu(buf[1]);
                if (!policydb_role_isvalid(p, ra->role) ||
                    !policydb_role_isvalid(p, ra->new_role))
                        goto bad;
                lra = ra;
        }

        rc = filename_trans_read(p, fp);
        if (rc)
                goto bad;

        rc = policydb_index(p);
        if (rc)
                goto bad;

        rc = -EINVAL;
        perm = string_to_av_perm(p, p->process_class, "transition");
        if (!perm) {
                pr_err("SELinux: process transition permission is required, not defined in policy\n");
                goto bad;
        }
        p->process_trans_perms = perm;
        perm = string_to_av_perm(p, p->process_class, "dyntransition");
        if (!perm) {
                pr_err("SELinux: process dyntransition permission is required, not defined in policy\n");
                goto bad;
        }
        p->process_trans_perms |= perm;

        rc = ocontext_read(p, info, fp);
        if (rc)
                goto bad;

        rc = genfs_read(p, fp);
        if (rc)
                goto bad;

        rc = range_read(p, fp);
        if (rc)
                goto bad;

        rc = -ENOMEM;
        p->type_attr_map_array = kvcalloc(
                p->p_types.nprim, sizeof(*p->type_attr_map_array), GFP_KERNEL);
        if (!p->type_attr_map_array)
                goto bad;

        /* just in case ebitmap_init() becomes more than just a memset(0): */
        for (i = 0; i < p->p_types.nprim; i++)
                ebitmap_init(&p->type_attr_map_array[i]);

        for (i = 0; i < p->p_types.nprim; i++) {
                struct ebitmap *e = &p->type_attr_map_array[i];

                if (p->policyvers >= POLICYDB_VERSION_AVTAB) {
                        rc = ebitmap_read(e, fp);
                        if (rc)
                                goto bad;
                }
                /* add the type itself as the degenerate case */
                rc = ebitmap_set_bit(e, i, 1);
                if (rc)
                        goto bad;
        }

        rc = policydb_bounds_sanity_check(p);
        if (rc)
                goto bad;

        rc = 0;
out:
        return rc;
bad:
        kfree(rtk);
        kfree(rtd);
        policydb_destroy(p);
        goto out;
}

/*
 * Write a MLS level structure to a policydb binary
 * representation file.
 */
static int mls_write_level(struct mls_level *l, struct policy_file *fp)
{
        __le32 buf[1];
        int rc;

        buf[0] = cpu_to_le32(l->sens);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;

        rc = ebitmap_write(&l->cat, fp);
        if (rc)
                return rc;

        return 0;
}

/*
 * Write a MLS range structure to a policydb binary
 * representation file.
 */
static int mls_write_range_helper(struct mls_range *r, struct policy_file *fp)
{
        __le32 buf[3];
        size_t items;
        int rc, eq;

        eq = mls_level_eq(&r->level[1], &r->level[0]);

        if (eq)
                items = 2;
        else
                items = 3;
        buf[0] = cpu_to_le32(items - 1);
        buf[1] = cpu_to_le32(r->level[0].sens);
        if (!eq)
                buf[2] = cpu_to_le32(r->level[1].sens);

        BUG_ON(items > ARRAY_SIZE(buf));

        rc = put_entry(buf, sizeof(u32), items, fp);
        if (rc)
                return rc;

        rc = ebitmap_write(&r->level[0].cat, fp);
        if (rc)
                return rc;
        if (!eq) {
                rc = ebitmap_write(&r->level[1].cat, fp);
                if (rc)
                        return rc;
        }

        return 0;
}

static int sens_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct level_datum *levdatum = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        __le32 buf[2];
        size_t len;
        int rc;

        len = strlen(key);
        buf[0] = cpu_to_le32(len);
        buf[1] = cpu_to_le32(levdatum->isalias);
        rc = put_entry(buf, sizeof(u32), 2, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        rc = mls_write_level(&levdatum->level, fp);
        if (rc)
                return rc;

        return 0;
}

static int cat_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct cat_datum *catdatum = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        __le32 buf[3];
        size_t len;
        int rc;

        len = strlen(key);
        buf[0] = cpu_to_le32(len);
        buf[1] = cpu_to_le32(catdatum->value);
        buf[2] = cpu_to_le32(catdatum->isalias);
        rc = put_entry(buf, sizeof(u32), 3, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        return 0;
}

static int role_trans_write_one(void *key, void *datum, void *ptr)
{
        struct role_trans_key *rtk = key;
        struct role_trans_datum *rtd = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        struct policydb *p = pd->p;
        __le32 buf[3];
        int rc;

        buf[0] = cpu_to_le32(rtk->role);
        buf[1] = cpu_to_le32(rtk->type);
        buf[2] = cpu_to_le32(rtd->new_role);
        rc = put_entry(buf, sizeof(u32), 3, fp);
        if (rc)
                return rc;
        if (p->policyvers >= POLICYDB_VERSION_ROLETRANS) {
                buf[0] = cpu_to_le32(rtk->tclass);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;
        }
        return 0;
}

static int role_trans_write(struct policydb *p, struct policy_file *fp)
{
        struct policy_data pd = { .p = p, .fp = fp };
        __le32 buf[1];
        int rc;

        buf[0] = cpu_to_le32(p->role_tr.nel);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;

        return hashtab_map(&p->role_tr, role_trans_write_one, &pd);
}

static int role_allow_write(struct role_allow *r, struct policy_file *fp)
{
        struct role_allow *ra;
        __le32 buf[2];
        size_t nel;
        int rc;

        nel = 0;
        for (ra = r; ra; ra = ra->next)
                nel++;
        buf[0] = cpu_to_le32(nel);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;
        for (ra = r; ra; ra = ra->next) {
                buf[0] = cpu_to_le32(ra->role);
                buf[1] = cpu_to_le32(ra->new_role);
                rc = put_entry(buf, sizeof(u32), 2, fp);
                if (rc)
                        return rc;
        }
        return 0;
}

/*
 * Write a security context structure
 * to a policydb binary representation file.
 */
static int context_write(struct policydb *p, struct context *c, struct policy_file *fp)
{
        int rc;
        __le32 buf[3];

        buf[0] = cpu_to_le32(c->user);
        buf[1] = cpu_to_le32(c->role);
        buf[2] = cpu_to_le32(c->type);

        rc = put_entry(buf, sizeof(u32), 3, fp);
        if (rc)
                return rc;

        rc = mls_write_range_helper(&c->range, fp);
        if (rc)
                return rc;

        return 0;
}

/*
 * The following *_write functions are used to
 * write the symbol data to a policy database
 * binary representation file.
 */

static int perm_write(void *vkey, void *datum, void *fp)
{
        char *key = vkey;
        struct perm_datum *perdatum = datum;
        __le32 buf[2];
        size_t len;
        int rc;

        len = strlen(key);
        buf[0] = cpu_to_le32(len);
        buf[1] = cpu_to_le32(perdatum->value);
        rc = put_entry(buf, sizeof(u32), 2, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        return 0;
}

static int common_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct common_datum *comdatum = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        __le32 buf[4];
        size_t len;
        int rc;

        len = strlen(key);
        buf[0] = cpu_to_le32(len);
        buf[1] = cpu_to_le32(comdatum->value);
        buf[2] = cpu_to_le32(comdatum->permissions.nprim);
        buf[3] = cpu_to_le32(comdatum->permissions.table.nel);
        rc = put_entry(buf, sizeof(u32), 4, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        rc = hashtab_map(&comdatum->permissions.table, perm_write, fp);
        if (rc)
                return rc;

        return 0;
}

static int type_set_write(struct type_set *t, struct policy_file *fp)
{
        int rc;
        __le32 buf[1];

        if (ebitmap_write(&t->types, fp))
                return -EINVAL;
        if (ebitmap_write(&t->negset, fp))
                return -EINVAL;

        buf[0] = cpu_to_le32(t->flags);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return -EINVAL;

        return 0;
}

static int write_cons_helper(struct policydb *p, struct constraint_node *node,
                             struct policy_file *fp)
{
        struct constraint_node *c;
        struct constraint_expr *e;
        __le32 buf[3];
        u32 nel;
        int rc;

        for (c = node; c; c = c->next) {
                nel = 0;
                for (e = c->expr; e; e = e->next)
                        nel++;
                buf[0] = cpu_to_le32(c->permissions);
                buf[1] = cpu_to_le32(nel);
                rc = put_entry(buf, sizeof(u32), 2, fp);
                if (rc)
                        return rc;
                for (e = c->expr; e; e = e->next) {
                        buf[0] = cpu_to_le32(e->expr_type);
                        buf[1] = cpu_to_le32(e->attr);
                        buf[2] = cpu_to_le32(e->op);
                        rc = put_entry(buf, sizeof(u32), 3, fp);
                        if (rc)
                                return rc;

                        switch (e->expr_type) {
                        case CEXPR_NAMES:
                                rc = ebitmap_write(&e->names, fp);
                                if (rc)
                                        return rc;
                                if (p->policyvers >=
                                    POLICYDB_VERSION_CONSTRAINT_NAMES) {
                                        rc = type_set_write(e->type_names, fp);
                                        if (rc)
                                                return rc;
                                }
                                break;
                        default:
                                break;
                        }
                }
        }

        return 0;
}

static int class_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct class_datum *cladatum = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        struct policydb *p = pd->p;
        struct constraint_node *c;
        __le32 buf[6];
        u32 ncons;
        size_t len, len2;
        int rc;

        len = strlen(key);
        if (cladatum->comkey)
                len2 = strlen(cladatum->comkey);
        else
                len2 = 0;

        ncons = 0;
        for (c = cladatum->constraints; c; c = c->next)
                ncons++;

        buf[0] = cpu_to_le32(len);
        buf[1] = cpu_to_le32(len2);
        buf[2] = cpu_to_le32(cladatum->value);
        buf[3] = cpu_to_le32(cladatum->permissions.nprim);
        buf[4] = cpu_to_le32(cladatum->permissions.table.nel);
        buf[5] = cpu_to_le32(ncons);
        rc = put_entry(buf, sizeof(u32), 6, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        if (cladatum->comkey) {
                rc = put_entry(cladatum->comkey, 1, len2, fp);
                if (rc)
                        return rc;
        }

        rc = hashtab_map(&cladatum->permissions.table, perm_write, fp);
        if (rc)
                return rc;

        rc = write_cons_helper(p, cladatum->constraints, fp);
        if (rc)
                return rc;

        /* write out the validatetrans rule */
        ncons = 0;
        for (c = cladatum->validatetrans; c; c = c->next)
                ncons++;

        buf[0] = cpu_to_le32(ncons);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;

        rc = write_cons_helper(p, cladatum->validatetrans, fp);
        if (rc)
                return rc;

        if (p->policyvers >= POLICYDB_VERSION_NEW_OBJECT_DEFAULTS) {
                buf[0] = cpu_to_le32(cladatum->default_user);
                buf[1] = cpu_to_le32(cladatum->default_role);
                buf[2] = cpu_to_le32(cladatum->default_range);

                rc = put_entry(buf, sizeof(uint32_t), 3, fp);
                if (rc)
                        return rc;
        }

        if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) {
                buf[0] = cpu_to_le32(cladatum->default_type);
                rc = put_entry(buf, sizeof(uint32_t), 1, fp);
                if (rc)
                        return rc;
        }

        return 0;
}

static int role_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct role_datum *role = datum;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        struct policydb *p = pd->p;
        __le32 buf[3];
        size_t items, len;
        int rc;

        len = strlen(key);
        items = 0;
        buf[items++] = cpu_to_le32(len);
        buf[items++] = cpu_to_le32(role->value);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                buf[items++] = cpu_to_le32(role->bounds);

        BUG_ON(items > ARRAY_SIZE(buf));

        rc = put_entry(buf, sizeof(u32), items, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        rc = ebitmap_write(&role->dominates, fp);
        if (rc)
                return rc;

        rc = ebitmap_write(&role->types, fp);
        if (rc)
                return rc;

        return 0;
}

static int type_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct type_datum *typdatum = datum;
        struct policy_data *pd = ptr;
        struct policydb *p = pd->p;
        struct policy_file *fp = pd->fp;
        __le32 buf[4];
        int rc;
        size_t items, len;

        len = strlen(key);
        items = 0;
        buf[items++] = cpu_to_le32(len);
        buf[items++] = cpu_to_le32(typdatum->value);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY) {
                u32 properties = 0;

                if (typdatum->primary)
                        properties |= TYPEDATUM_PROPERTY_PRIMARY;

                if (typdatum->attribute)
                        properties |= TYPEDATUM_PROPERTY_ATTRIBUTE;

                buf[items++] = cpu_to_le32(properties);
                buf[items++] = cpu_to_le32(typdatum->bounds);
        } else {
                buf[items++] = cpu_to_le32(typdatum->primary);
        }
        BUG_ON(items > ARRAY_SIZE(buf));
        rc = put_entry(buf, sizeof(u32), items, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        return 0;
}

static int user_write(void *vkey, void *datum, void *ptr)
{
        char *key = vkey;
        struct user_datum *usrdatum = datum;
        struct policy_data *pd = ptr;
        struct policydb *p = pd->p;
        struct policy_file *fp = pd->fp;
        __le32 buf[3];
        size_t items, len;
        int rc;

        len = strlen(key);
        items = 0;
        buf[items++] = cpu_to_le32(len);
        buf[items++] = cpu_to_le32(usrdatum->value);
        if (p->policyvers >= POLICYDB_VERSION_BOUNDARY)
                buf[items++] = cpu_to_le32(usrdatum->bounds);
        BUG_ON(items > ARRAY_SIZE(buf));
        rc = put_entry(buf, sizeof(u32), items, fp);
        if (rc)
                return rc;

        rc = put_entry(key, 1, len, fp);
        if (rc)
                return rc;

        rc = ebitmap_write(&usrdatum->roles, fp);
        if (rc)
                return rc;

        rc = mls_write_range_helper(&usrdatum->range, fp);
        if (rc)
                return rc;

        rc = mls_write_level(&usrdatum->dfltlevel, fp);
        if (rc)
                return rc;

        return 0;
}

/* clang-format off */
static int (*const write_f[SYM_NUM])(void *key, void *datum, void *datap) = {
        common_write,
        class_write,
        role_write,
        type_write,
        user_write,
        cond_write_bool,
        sens_write,
        cat_write,
};
/* clang-format on */

static int ocontext_write(struct policydb *p,
                          const struct policydb_compat_info *info,
                          struct policy_file *fp)
{
        unsigned int i, j;
        int rc;
        size_t nel, len;
        __be64 prefixbuf[1];
        __le32 buf[3];
        u32 nodebuf[8];
        struct ocontext *c;
        for (i = 0; i < info->ocon_num; i++) {
                nel = 0;
                for (c = p->ocontexts[i]; c; c = c->next)
                        nel++;
                buf[0] = cpu_to_le32(nel);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;
                for (c = p->ocontexts[i]; c; c = c->next) {
                        switch (i) {
                        case OCON_ISID:
                                buf[0] = cpu_to_le32(c->sid[0]);
                                rc = put_entry(buf, sizeof(u32), 1, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_FS:
                        case OCON_NETIF:
                                len = strlen(c->u.name);
                                buf[0] = cpu_to_le32(len);
                                rc = put_entry(buf, sizeof(u32), 1, fp);
                                if (rc)
                                        return rc;
                                rc = put_entry(c->u.name, 1, len, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[1], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_PORT:
                                buf[0] = cpu_to_le32(c->u.port.protocol);
                                buf[1] = cpu_to_le32(c->u.port.low_port);
                                buf[2] = cpu_to_le32(c->u.port.high_port);
                                rc = put_entry(buf, sizeof(u32), 3, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_NODE:
                                nodebuf[0] = c->u.node.addr; /* network order */
                                nodebuf[1] = c->u.node.mask; /* network order */
                                rc = put_entry(nodebuf, sizeof(u32), 2, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_FSUSE:
                                buf[0] = cpu_to_le32(c->v.behavior);
                                len = strlen(c->u.name);
                                buf[1] = cpu_to_le32(len);
                                rc = put_entry(buf, sizeof(u32), 2, fp);
                                if (rc)
                                        return rc;
                                rc = put_entry(c->u.name, 1, len, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_NODE6:
                                for (j = 0; j < 4; j++)
                                        nodebuf[j] =
                                                c->u.node6.addr
                                                        [j]; /* network order */
                                for (j = 0; j < 4; j++)
                                        nodebuf[j + 4] =
                                                c->u.node6.mask
                                                        [j]; /* network order */
                                rc = put_entry(nodebuf, sizeof(u32), 8, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_IBPKEY:
                                /* subnet_prefix is in CPU order */
                                prefixbuf[0] =
                                        cpu_to_be64(c->u.ibpkey.subnet_prefix);

                                rc = put_entry(prefixbuf, sizeof(u64), 1, fp);
                                if (rc)
                                        return rc;

                                buf[0] = cpu_to_le32(c->u.ibpkey.low_pkey);
                                buf[1] = cpu_to_le32(c->u.ibpkey.high_pkey);

                                rc = put_entry(buf, sizeof(u32), 2, fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        case OCON_IBENDPORT:
                                len = strlen(c->u.ibendport.dev_name);
                                buf[0] = cpu_to_le32(len);
                                buf[1] = cpu_to_le32(c->u.ibendport.port);
                                rc = put_entry(buf, sizeof(u32), 2, fp);
                                if (rc)
                                        return rc;
                                rc = put_entry(c->u.ibendport.dev_name, 1, len,
                                               fp);
                                if (rc)
                                        return rc;
                                rc = context_write(p, &c->context[0], fp);
                                if (rc)
                                        return rc;
                                break;
                        }
                }
        }
        return 0;
}

static int genfs_write(struct policydb *p, struct policy_file *fp)
{
        struct genfs *genfs;
        struct ocontext *c;
        size_t len;
        __le32 buf[1];
        int rc;

        len = 0;
        for (genfs = p->genfs; genfs; genfs = genfs->next)
                len++;
        buf[0] = cpu_to_le32(len);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;
        for (genfs = p->genfs; genfs; genfs = genfs->next) {
                len = strlen(genfs->fstype);
                buf[0] = cpu_to_le32(len);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;
                rc = put_entry(genfs->fstype, 1, len, fp);
                if (rc)
                        return rc;
                len = 0;
                for (c = genfs->head; c; c = c->next)
                        len++;
                buf[0] = cpu_to_le32(len);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;
                for (c = genfs->head; c; c = c->next) {
                        len = strlen(c->u.name);
                        buf[0] = cpu_to_le32(len);
                        rc = put_entry(buf, sizeof(u32), 1, fp);
                        if (rc)
                                return rc;
                        rc = put_entry(c->u.name, 1, len, fp);
                        if (rc)
                                return rc;
                        buf[0] = cpu_to_le32(c->v.sclass);
                        rc = put_entry(buf, sizeof(u32), 1, fp);
                        if (rc)
                                return rc;
                        rc = context_write(p, &c->context[0], fp);
                        if (rc)
                                return rc;
                }
        }
        return 0;
}

static int range_write_helper(void *key, void *data, void *ptr)
{
        __le32 buf[2];
        struct range_trans *rt = key;
        struct mls_range *r = data;
        struct policy_data *pd = ptr;
        struct policy_file *fp = pd->fp;
        struct policydb *p = pd->p;
        int rc;

        buf[0] = cpu_to_le32(rt->source_type);
        buf[1] = cpu_to_le32(rt->target_type);
        rc = put_entry(buf, sizeof(u32), 2, fp);
        if (rc)
                return rc;
        if (p->policyvers >= POLICYDB_VERSION_RANGETRANS) {
                buf[0] = cpu_to_le32(rt->target_class);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;
        }
        rc = mls_write_range_helper(r, fp);
        if (rc)
                return rc;

        return 0;
}

static int range_write(struct policydb *p, struct policy_file *fp)
{
        __le32 buf[1];
        int rc;
        struct policy_data pd;

        pd.p = p;
        pd.fp = fp;

        buf[0] = cpu_to_le32(p->range_tr.nel);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;

        /* actually write all of the entries */
        rc = hashtab_map(&p->range_tr, range_write_helper, &pd);
        if (rc)
                return rc;

        return 0;
}

static int filename_write_helper_compat(void *key, void *data, void *ptr)
{
        struct filename_trans_key *ft = key;
        struct filename_trans_datum *datum = data;
        struct ebitmap_node *node;
        struct policy_file *fp = ptr;
        __le32 buf[4];
        int rc;
        u32 bit, len = strlen(ft->name);

        do {
                ebitmap_for_each_positive_bit(&datum->stypes, node, bit)
                {
                        buf[0] = cpu_to_le32(len);
                        rc = put_entry(buf, sizeof(u32), 1, fp);
                        if (rc)
                                return rc;

                        rc = put_entry(ft->name, sizeof(char), len, fp);
                        if (rc)
                                return rc;

                        buf[0] = cpu_to_le32(bit + 1);
                        buf[1] = cpu_to_le32(ft->ttype);
                        buf[2] = cpu_to_le32(ft->tclass);
                        buf[3] = cpu_to_le32(datum->otype);

                        rc = put_entry(buf, sizeof(u32), 4, fp);
                        if (rc)
                                return rc;
                }

                datum = datum->next;
        } while (unlikely(datum));

        return 0;
}

static int filename_write_helper(void *key, void *data, void *ptr)
{
        struct filename_trans_key *ft = key;
        struct filename_trans_datum *datum;
        struct policy_file *fp = ptr;
        __le32 buf[3];
        int rc;
        u32 ndatum, len = strlen(ft->name);

        buf[0] = cpu_to_le32(len);
        rc = put_entry(buf, sizeof(u32), 1, fp);
        if (rc)
                return rc;

        rc = put_entry(ft->name, sizeof(char), len, fp);
        if (rc)
                return rc;

        ndatum = 0;
        datum = data;
        do {
                ndatum++;
                datum = datum->next;
        } while (unlikely(datum));

        buf[0] = cpu_to_le32(ft->ttype);
        buf[1] = cpu_to_le32(ft->tclass);
        buf[2] = cpu_to_le32(ndatum);
        rc = put_entry(buf, sizeof(u32), 3, fp);
        if (rc)
                return rc;

        datum = data;
        do {
                rc = ebitmap_write(&datum->stypes, fp);
                if (rc)
                        return rc;

                buf[0] = cpu_to_le32(datum->otype);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;

                datum = datum->next;
        } while (unlikely(datum));

        return 0;
}

static int filename_trans_write(struct policydb *p, struct policy_file *fp)
{
        __le32 buf[1];
        int rc;

        if (p->policyvers < POLICYDB_VERSION_FILENAME_TRANS)
                return 0;

        if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) {
                buf[0] = cpu_to_le32(p->compat_filename_trans_count);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;

                rc = hashtab_map(&p->filename_trans,
                                 filename_write_helper_compat, fp);
        } else {
                buf[0] = cpu_to_le32(p->filename_trans.nel);
                rc = put_entry(buf, sizeof(u32), 1, fp);
                if (rc)
                        return rc;

                rc = hashtab_map(&p->filename_trans, filename_write_helper, fp);
        }
        return rc;
}

/*
 * Write the configuration data in a policy database
 * structure to a policy database binary representation
 * file.
 */
int policydb_write(struct policydb *p, struct policy_file *fp)
{
        unsigned int num_syms;
        int rc;
        __le32 buf[4];
        u32 config, i;
        size_t len;
        const struct policydb_compat_info *info;

        /*
         * refuse to write policy older than compressed avtab
         * to simplify the writer.  There are other tests dropped
         * since we assume this throughout the writer code.  Be
         * careful if you ever try to remove this restriction
         */
        if (p->policyvers < POLICYDB_VERSION_AVTAB) {
                pr_err("SELinux: refusing to write policy version %d."
                       "  Because it is less than version %d\n",
                       p->policyvers, POLICYDB_VERSION_AVTAB);
                return -EINVAL;
        }

        config = 0;
        if (p->mls_enabled)
                config |= POLICYDB_CONFIG_MLS;

        if (p->reject_unknown)
                config |= REJECT_UNKNOWN;
        if (p->allow_unknown)
                config |= ALLOW_UNKNOWN;

        /* Write the magic number and string identifiers. */
        buf[0] = cpu_to_le32(POLICYDB_MAGIC);
        len = strlen(POLICYDB_STRING);
        buf[1] = cpu_to_le32(len);
        rc = put_entry(buf, sizeof(u32), 2, fp);
        if (rc)
                return rc;
        rc = put_entry(POLICYDB_STRING, 1, len, fp);
        if (rc)
                return rc;

        /* Write the version, config, and table sizes. */
        info = policydb_lookup_compat(p->policyvers);
        if (!info) {
                pr_err("SELinux: compatibility lookup failed for policy "
                       "version %d\n",
                       p->policyvers);
                return -EINVAL;
        }

        buf[0] = cpu_to_le32(p->policyvers);
        buf[1] = cpu_to_le32(config);
        buf[2] = cpu_to_le32(info->sym_num);
        buf[3] = cpu_to_le32(info->ocon_num);

        rc = put_entry(buf, sizeof(u32), 4, fp);
        if (rc)
                return rc;

        if (p->policyvers >= POLICYDB_VERSION_POLCAP) {
                rc = ebitmap_write(&p->policycaps, fp);
                if (rc)
                        return rc;
        }

        if (p->policyvers >= POLICYDB_VERSION_PERMISSIVE) {
                rc = ebitmap_write(&p->permissive_map, fp);
                if (rc)
                        return rc;
        }

        num_syms = info->sym_num;
        for (i = 0; i < num_syms; i++) {
                struct policy_data pd;

                pd.fp = fp;
                pd.p = p;

                buf[0] = cpu_to_le32(p->symtab[i].nprim);
                buf[1] = cpu_to_le32(p->symtab[i].table.nel);

                rc = put_entry(buf, sizeof(u32), 2, fp);
                if (rc)
                        return rc;
                rc = hashtab_map(&p->symtab[i].table, write_f[i], &pd);
                if (rc)
                        return rc;
        }

        rc = avtab_write(p, &p->te_avtab, fp);
        if (rc)
                return rc;

        rc = cond_write_list(p, fp);
        if (rc)
                return rc;

        rc = role_trans_write(p, fp);
        if (rc)
                return rc;

        rc = role_allow_write(p->role_allow, fp);
        if (rc)
                return rc;

        rc = filename_trans_write(p, fp);
        if (rc)
                return rc;

        rc = ocontext_write(p, info, fp);
        if (rc)
                return rc;

        rc = genfs_write(p, fp);
        if (rc)
                return rc;

        rc = range_write(p, fp);
        if (rc)
                return rc;

        for (i = 0; i < p->p_types.nprim; i++) {
                struct ebitmap *e = &p->type_attr_map_array[i];

                rc = ebitmap_write(e, fp);
                if (rc)
                        return rc;
        }

        return 0;
}









































 1798 


 1793 



 1789 
















 1783 









 1794 

































































 1785 








 1797 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
// SPDX-License-Identifier: GPL-2.0

#include <linux/compiler.h>
#include <linux/context_tracking.h>
#include <linux/errno.h>
#include <linux/nospec.h>
#include <linux/ptrace.h>
#include <linux/randomize_kstack.h>
#include <linux/syscalls.h>

#include <asm/debug-monitors.h>
#include <asm/exception.h>
#include <asm/fpsimd.h>
#include <asm/syscall.h>
#include <asm/thread_info.h>
#include <asm/unistd.h>
#include <asm/unistd_compat_32.h>

long compat_arm_syscall(struct pt_regs *regs, int scno);
long sys_ni_syscall(void);

static long do_ni_syscall(struct pt_regs *regs, int scno)
{
        if (is_compat_task()) {
                long ret = compat_arm_syscall(regs, scno);
                if (ret != -ENOSYS)
                        return ret;
        }

        return sys_ni_syscall();
}

static long __invoke_syscall(struct pt_regs *regs, syscall_fn_t syscall_fn)
{
        return syscall_fn(regs);
}

static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
                           unsigned int sc_nr,
                           const syscall_fn_t syscall_table[])
{
        long ret;

        add_random_kstack_offset();

        if (scno < sc_nr) {
                syscall_fn_t syscall_fn;
                syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)];
                ret = __invoke_syscall(regs, syscall_fn);
        } else {
                ret = do_ni_syscall(regs, scno);
        }

        syscall_set_return_value(current, regs, 0, ret);

        /*
         * This value will get limited by KSTACK_OFFSET_MAX(), which is 10
         * bits. The actual entropy will be further reduced by the compiler
         * when applying stack alignment constraints: the AAPCS mandates a
         * 16-byte aligned SP at function boundaries, which will remove the
         * 4 low bits from any entropy chosen here.
         *
         * The resulting 6 bits of entropy is seen in SP[9:4].
         */
        choose_random_kstack_offset(get_random_u16());
}

static inline bool has_syscall_work(unsigned long flags)
{
        return unlikely(flags & _TIF_SYSCALL_WORK);
}

static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
                           const syscall_fn_t syscall_table[])
{
        unsigned long flags = read_thread_flags();

        regs->orig_x0 = regs->regs[0];
        regs->syscallno = scno;

        /*
         * BTI note:
         * The architecture does not guarantee that SPSR.BTYPE is zero
         * on taking an SVC, so we could return to userspace with a
         * non-zero BTYPE after the syscall.
         *
         * This shouldn't matter except when userspace is explicitly
         * doing something stupid, such as setting PROT_BTI on a page
         * that lacks conforming BTI/PACIxSP instructions, falling
         * through from one executable page to another with differing
         * PROT_BTI, or messing with BTYPE via ptrace: in such cases,
         * userspace should not be surprised if a SIGILL occurs on
         * syscall return.
         *
         * So, don't touch regs->pstate & PSR_BTYPE_MASK here.
         * (Similarly for HVC and SMC elsewhere.)
         */

        if (flags & _TIF_MTE_ASYNC_FAULT) {
                /*
                 * Process the asynchronous tag check fault before the actual
                 * syscall. do_notify_resume() will send a signal to userspace
                 * before the syscall is restarted.
                 */
                syscall_set_return_value(current, regs, -ERESTARTNOINTR, 0);
                return;
        }

        if (has_syscall_work(flags)) {
                /*
                 * The de-facto standard way to skip a system call using ptrace
                 * is to set the system call to -1 (NO_SYSCALL) and set x0 to a
                 * suitable error code for consumption by userspace. However,
                 * this cannot be distinguished from a user-issued syscall(-1)
                 * and so we must set x0 to -ENOSYS here in case the tracer doesn't
                 * issue the skip and we fall into trace_exit with x0 preserved.
                 *
                 * This is slightly odd because it also means that if a tracer
                 * sets the system call number to -1 but does not initialise x0,
                 * then x0 will be preserved for all system calls apart from a
                 * user-issued syscall(-1). However, requesting a skip and not
                 * setting the return value is unlikely to do anything sensible
                 * anyway.
                 */
                if (scno == NO_SYSCALL)
                        syscall_set_return_value(current, regs, -ENOSYS, 0);
                scno = syscall_trace_enter(regs);
                if (scno == NO_SYSCALL)
                        goto trace_exit;
        }

        invoke_syscall(regs, scno, sc_nr, syscall_table);

        /*
         * The tracing status may have changed under our feet, so we have to
         * check again. However, if we were tracing entry, then we always trace
         * exit regardless, as the old entry assembly did.
         */
        if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
                flags = read_thread_flags();
                if (!has_syscall_work(flags) && !(flags & _TIF_SINGLESTEP))
                        return;
        }

trace_exit:
        syscall_trace_exit(regs);
}

void do_el0_svc(struct pt_regs *regs)
{
        el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table);
}

#ifdef CONFIG_COMPAT
void do_el0_svc_compat(struct pt_regs *regs)
{
        el0_svc_common(regs, regs->regs[7], __NR_compat32_syscalls,
                       compat_sys_call_table);
}
#endif






















    2 
























    2 
























   41 





























   63 




































    9 


















    9 















   16 




    2 


    1 





    2 




    1 




    3 

    9 


























   59 







   59 
    9 


   65 
   41 







    2 

















   66 








   65 
    1 





    2 






   17 

    1 



    1 























































    1 



    1 
    1 



    1 





    2 













   32 



















  269 










   23 




    4 




    2 


    2 
    2 













    1 


    2 
















    3 









    2 














   11 
    1 



    1 

    2 




    4 





    1 


    1 


    1 





   10 


































    3 




    3 











   30 




    1 

    2 


   11 






   11 

    1 

    1 






    5 

    5 







    2 


    2 





    1 




    1 







    1 
























    6 








    1 









   17 


    1 

   16 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2019 Arm Ltd.

#include <linux/arm-smccc.h>
#include <linux/kvm_host.h>

#include <asm/kvm_emulate.h>

#include <kvm/arm_hypercalls.h>
#include <kvm/arm_psci.h>

#define KVM_ARM_SMCCC_STD_FEATURES                                \
        GENMASK(KVM_REG_ARM_STD_BMAP_BIT_COUNT - 1, 0)
#define KVM_ARM_SMCCC_STD_HYP_FEATURES                                \
        GENMASK(KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT - 1, 0)
#define KVM_ARM_SMCCC_VENDOR_HYP_FEATURES                        \
        GENMASK(KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT - 1, 0)
#define KVM_ARM_SMCCC_VENDOR_HYP_FEATURES_2                        \
        GENMASK(KVM_REG_ARM_VENDOR_HYP_BMAP_2_BIT_COUNT - 1, 0)

static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val)
{
        struct system_time_snapshot systime_snapshot;
        u64 cycles = ~0UL;
        u32 feature;

        /*
         * system time and counter value must captured at the same
         * time to keep consistency and precision.
         */
        ktime_get_snapshot(&systime_snapshot);

        /*
         * This is only valid if the current clocksource is the
         * architected counter, as this is the only one the guest
         * can see.
         */
        if (systime_snapshot.cs_id != CSID_ARM_ARCH_COUNTER)
                return;

        /*
         * The guest selects one of the two reference counters
         * (virtual or physical) with the first argument of the SMCCC
         * call. In case the identifier is not supported, error out.
         */
        feature = smccc_get_arg1(vcpu);
        switch (feature) {
        case KVM_PTP_VIRT_COUNTER:
                cycles = systime_snapshot.cycles - vcpu->kvm->arch.timer_data.voffset;
                break;
        case KVM_PTP_PHYS_COUNTER:
                cycles = systime_snapshot.cycles - vcpu->kvm->arch.timer_data.poffset;
                break;
        default:
                return;
        }

        /*
         * This relies on the top bit of val[0] never being set for
         * valid values of system time, because that is *really* far
         * in the future (about 292 years from 1970, and at that stage
         * nobody will give a damn about it).
         */
        val[0] = upper_32_bits(systime_snapshot.real);
        val[1] = lower_32_bits(systime_snapshot.real);
        val[2] = upper_32_bits(cycles);
        val[3] = lower_32_bits(cycles);
}

static bool kvm_smccc_default_allowed(u32 func_id)
{
        switch (func_id) {
        /*
         * List of function-ids that are not gated with the bitmapped
         * feature firmware registers, and are to be allowed for
         * servicing the call by default.
         */
        case ARM_SMCCC_VERSION_FUNC_ID:
        case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
                return true;
        default:
                /* PSCI 0.2 and up is in the 0:0x1f range */
                if (ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD &&
                    ARM_SMCCC_FUNC_NUM(func_id) <= 0x1f)
                        return true;

                /*
                 * KVM's PSCI 0.1 doesn't comply with SMCCC, and has
                 * its own function-id base and range
                 */
                if (func_id >= KVM_PSCI_FN(0) && func_id <= KVM_PSCI_FN(3))
                        return true;

                return false;
        }
}

static bool kvm_smccc_test_fw_bmap(struct kvm_vcpu *vcpu, u32 func_id)
{
        struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;

        switch (func_id) {
        case ARM_SMCCC_TRNG_VERSION:
        case ARM_SMCCC_TRNG_FEATURES:
        case ARM_SMCCC_TRNG_GET_UUID:
        case ARM_SMCCC_TRNG_RND32:
        case ARM_SMCCC_TRNG_RND64:
                return test_bit(KVM_REG_ARM_STD_BIT_TRNG_V1_0,
                                &smccc_feat->std_bmap);
        case ARM_SMCCC_HV_PV_TIME_FEATURES:
        case ARM_SMCCC_HV_PV_TIME_ST:
                return test_bit(KVM_REG_ARM_STD_HYP_BIT_PV_TIME,
                                &smccc_feat->std_hyp_bmap);
        case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID:
        case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID:
                return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT,
                                &smccc_feat->vendor_hyp_bmap);
        case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID:
                return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PTP,
                                &smccc_feat->vendor_hyp_bmap);
        default:
                return false;
        }
}

#define SMC32_ARCH_RANGE_BEGIN        ARM_SMCCC_VERSION_FUNC_ID
#define SMC32_ARCH_RANGE_END        ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                \
                                                   ARM_SMCCC_SMC_32,                \
                                                   0, ARM_SMCCC_FUNC_MASK)

#define SMC64_ARCH_RANGE_BEGIN        ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                \
                                                   ARM_SMCCC_SMC_64,                \
                                                   0, 0)
#define SMC64_ARCH_RANGE_END        ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                \
                                                   ARM_SMCCC_SMC_64,                \
                                                   0, ARM_SMCCC_FUNC_MASK)

static int kvm_smccc_filter_insert_reserved(struct kvm *kvm)
{
        int r;

        /*
         * Prevent userspace from handling any SMCCC calls in the architecture
         * range, avoiding the risk of misrepresenting Spectre mitigation status
         * to the guest.
         */
        r = mtree_insert_range(&kvm->arch.smccc_filter,
                               SMC32_ARCH_RANGE_BEGIN, SMC32_ARCH_RANGE_END,
                               xa_mk_value(KVM_SMCCC_FILTER_HANDLE),
                               GFP_KERNEL_ACCOUNT);
        if (r)
                goto out_destroy;

        r = mtree_insert_range(&kvm->arch.smccc_filter,
                               SMC64_ARCH_RANGE_BEGIN, SMC64_ARCH_RANGE_END,
                               xa_mk_value(KVM_SMCCC_FILTER_HANDLE),
                               GFP_KERNEL_ACCOUNT);
        if (r)
                goto out_destroy;

        return 0;
out_destroy:
        mtree_destroy(&kvm->arch.smccc_filter);
        return r;
}

static bool kvm_smccc_filter_configured(struct kvm *kvm)
{
        return !mtree_empty(&kvm->arch.smccc_filter);
}

static int kvm_smccc_set_filter(struct kvm *kvm, struct kvm_smccc_filter __user *uaddr)
{
        const void *zero_page = page_to_virt(ZERO_PAGE(0));
        struct kvm_smccc_filter filter;
        u32 start, end;
        int r;

        if (copy_from_user(&filter, uaddr, sizeof(filter)))
                return -EFAULT;

        if (memcmp(filter.pad, zero_page, sizeof(filter.pad)))
                return -EINVAL;

        start = filter.base;
        end = start + filter.nr_functions - 1;

        if (end < start || filter.action >= NR_SMCCC_FILTER_ACTIONS)
                return -EINVAL;

        mutex_lock(&kvm->arch.config_lock);

        if (kvm_vm_has_ran_once(kvm)) {
                r = -EBUSY;
                goto out_unlock;
        }

        if (!kvm_smccc_filter_configured(kvm)) {
                r = kvm_smccc_filter_insert_reserved(kvm);
                if (WARN_ON_ONCE(r))
                        goto out_unlock;
        }

        r = mtree_insert_range(&kvm->arch.smccc_filter, start, end,
                               xa_mk_value(filter.action), GFP_KERNEL_ACCOUNT);
out_unlock:
        mutex_unlock(&kvm->arch.config_lock);
        return r;
}

static u8 kvm_smccc_filter_get_action(struct kvm *kvm, u32 func_id)
{
        unsigned long idx = func_id;
        void *val;

        if (!kvm_smccc_filter_configured(kvm))
                return KVM_SMCCC_FILTER_HANDLE;

        /*
         * But where's the error handling, you say?
         *
         * mt_find() returns NULL if no entry was found, which just so happens
         * to match KVM_SMCCC_FILTER_HANDLE.
         */
        val = mt_find(&kvm->arch.smccc_filter, &idx, idx);
        return xa_to_value(val);
}

static u8 kvm_smccc_get_action(struct kvm_vcpu *vcpu, u32 func_id)
{
        /*
         * Intervening actions in the SMCCC filter take precedence over the
         * pseudo-firmware register bitmaps.
         */
        u8 action = kvm_smccc_filter_get_action(vcpu->kvm, func_id);
        if (action != KVM_SMCCC_FILTER_HANDLE)
                return action;

        if (kvm_smccc_test_fw_bmap(vcpu, func_id) ||
            kvm_smccc_default_allowed(func_id))
                return KVM_SMCCC_FILTER_HANDLE;

        return KVM_SMCCC_FILTER_DENY;
}

static void kvm_prepare_hypercall_exit(struct kvm_vcpu *vcpu, u32 func_id)
{
        u8 ec = ESR_ELx_EC(kvm_vcpu_get_esr(vcpu));
        struct kvm_run *run = vcpu->run;
        u64 flags = 0;

        if (ec == ESR_ELx_EC_SMC32 || ec == ESR_ELx_EC_SMC64)
                flags |= KVM_HYPERCALL_EXIT_SMC;

        if (!kvm_vcpu_trap_il_is32bit(vcpu))
                flags |= KVM_HYPERCALL_EXIT_16BIT;

        run->exit_reason = KVM_EXIT_HYPERCALL;
        run->hypercall = (typeof(run->hypercall)) {
                .nr        = func_id,
                .flags        = flags,
        };
}

int kvm_smccc_call_handler(struct kvm_vcpu *vcpu)
{
        struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;
        u32 func_id = smccc_get_function(vcpu);
        u64 val[4] = {SMCCC_RET_NOT_SUPPORTED};
        u32 feature;
        u8 action;
        gpa_t gpa;
        uuid_t uuid;

        action = kvm_smccc_get_action(vcpu, func_id);
        switch (action) {
        case KVM_SMCCC_FILTER_HANDLE:
                break;
        case KVM_SMCCC_FILTER_DENY:
                goto out;
        case KVM_SMCCC_FILTER_FWD_TO_USER:
                kvm_prepare_hypercall_exit(vcpu, func_id);
                return 0;
        default:
                WARN_RATELIMIT(1, "Unhandled SMCCC filter action: %d\n", action);
                goto out;
        }

        switch (func_id) {
        case ARM_SMCCC_VERSION_FUNC_ID:
                val[0] = ARM_SMCCC_VERSION_1_1;
                break;
        case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
                feature = smccc_get_arg1(vcpu);
                switch (feature) {
                case ARM_SMCCC_ARCH_WORKAROUND_1:
                        switch (arm64_get_spectre_v2_state()) {
                        case SPECTRE_VULNERABLE:
                                break;
                        case SPECTRE_MITIGATED:
                                val[0] = SMCCC_RET_SUCCESS;
                                break;
                        case SPECTRE_UNAFFECTED:
                                val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
                                break;
                        }
                        break;
                case ARM_SMCCC_ARCH_WORKAROUND_2:
                        switch (arm64_get_spectre_v4_state()) {
                        case SPECTRE_VULNERABLE:
                                break;
                        case SPECTRE_MITIGATED:
                                /*
                                 * SSBS everywhere: Indicate no firmware
                                 * support, as the SSBS support will be
                                 * indicated to the guest and the default is
                                 * safe.
                                 *
                                 * Otherwise, expose a permanent mitigation
                                 * to the guest, and hide SSBS so that the
                                 * guest stays protected.
                                 */
                                if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP))
                                        break;
                                fallthrough;
                        case SPECTRE_UNAFFECTED:
                                val[0] = SMCCC_RET_NOT_REQUIRED;
                                break;
                        }
                        break;
                case ARM_SMCCC_ARCH_WORKAROUND_3:
                        switch (arm64_get_spectre_bhb_state()) {
                        case SPECTRE_VULNERABLE:
                                break;
                        case SPECTRE_MITIGATED:
                                val[0] = SMCCC_RET_SUCCESS;
                                break;
                        case SPECTRE_UNAFFECTED:
                                val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
                                break;
                        }
                        break;
                case ARM_SMCCC_HV_PV_TIME_FEATURES:
                        if (test_bit(KVM_REG_ARM_STD_HYP_BIT_PV_TIME,
                                     &smccc_feat->std_hyp_bmap))
                                val[0] = SMCCC_RET_SUCCESS;
                        break;
                }
                break;
        case ARM_SMCCC_HV_PV_TIME_FEATURES:
                val[0] = kvm_hypercall_pv_features(vcpu);
                break;
        case ARM_SMCCC_HV_PV_TIME_ST:
                gpa = kvm_init_stolen_time(vcpu);
                if (gpa != INVALID_GPA)
                        val[0] = gpa;
                break;
        case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID:
                uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM;
                val[0] = smccc_uuid_to_reg(&uuid, 0);
                val[1] = smccc_uuid_to_reg(&uuid, 1);
                val[2] = smccc_uuid_to_reg(&uuid, 2);
                val[3] = smccc_uuid_to_reg(&uuid, 3);
                break;
        case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID:
                val[0] = smccc_feat->vendor_hyp_bmap;
                /* Function numbers 2-63 are reserved for pKVM for now */
                val[2] = smccc_feat->vendor_hyp_bmap_2;
                break;
        case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID:
                kvm_ptp_get_time(vcpu, val);
                break;
        case ARM_SMCCC_TRNG_VERSION:
        case ARM_SMCCC_TRNG_FEATURES:
        case ARM_SMCCC_TRNG_GET_UUID:
        case ARM_SMCCC_TRNG_RND32:
        case ARM_SMCCC_TRNG_RND64:
                return kvm_trng_call(vcpu);
        default:
                return kvm_psci_call(vcpu);
        }

out:
        smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]);
        return 1;
}

static const u64 kvm_arm_fw_reg_ids[] = {
        KVM_REG_ARM_PSCI_VERSION,
        KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1,
        KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2,
        KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3,
        KVM_REG_ARM_STD_BMAP,
        KVM_REG_ARM_STD_HYP_BMAP,
        KVM_REG_ARM_VENDOR_HYP_BMAP,
        KVM_REG_ARM_VENDOR_HYP_BMAP_2,
};

void kvm_arm_init_hypercalls(struct kvm *kvm)
{
        struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat;

        smccc_feat->std_bmap = KVM_ARM_SMCCC_STD_FEATURES;
        smccc_feat->std_hyp_bmap = KVM_ARM_SMCCC_STD_HYP_FEATURES;
        smccc_feat->vendor_hyp_bmap = KVM_ARM_SMCCC_VENDOR_HYP_FEATURES;

        mt_init(&kvm->arch.smccc_filter);
}

void kvm_arm_teardown_hypercalls(struct kvm *kvm)
{
        mtree_destroy(&kvm->arch.smccc_filter);
}

int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu)
{
        return ARRAY_SIZE(kvm_arm_fw_reg_ids);
}

int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(kvm_arm_fw_reg_ids); i++) {
                if (put_user(kvm_arm_fw_reg_ids[i], uindices++))
                        return -EFAULT;
        }

        return 0;
}

#define KVM_REG_FEATURE_LEVEL_MASK        GENMASK(3, 0)

/*
 * Convert the workaround level into an easy-to-compare number, where higher
 * values mean better protection.
 */
static int get_kernel_wa_level(struct kvm_vcpu *vcpu, u64 regid)
{
        switch (regid) {
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
                switch (arm64_get_spectre_v2_state()) {
                case SPECTRE_VULNERABLE:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL;
                case SPECTRE_MITIGATED:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL;
                case SPECTRE_UNAFFECTED:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED;
                }
                return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL;
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
                switch (arm64_get_spectre_v4_state()) {
                case SPECTRE_MITIGATED:
                        /*
                         * As for the hypercall discovery, we pretend we
                         * don't have any FW mitigation if SSBS is there at
                         * all times.
                         */
                        if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP))
                                return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
                        fallthrough;
                case SPECTRE_UNAFFECTED:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED;
                case SPECTRE_VULNERABLE:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
                }
                break;
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
                switch (arm64_get_spectre_bhb_state()) {
                case SPECTRE_VULNERABLE:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
                case SPECTRE_MITIGATED:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL;
                case SPECTRE_UNAFFECTED:
                        return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED;
                }
                return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL;
        }

        return -EINVAL;
}

int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
{
        struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat;
        void __user *uaddr = (void __user *)(long)reg->addr;
        u64 val;

        switch (reg->id) {
        case KVM_REG_ARM_PSCI_VERSION:
                val = kvm_psci_version(vcpu);
                break;
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
                val = get_kernel_wa_level(vcpu, reg->id) & KVM_REG_FEATURE_LEVEL_MASK;
                break;
        case KVM_REG_ARM_STD_BMAP:
                val = READ_ONCE(smccc_feat->std_bmap);
                break;
        case KVM_REG_ARM_STD_HYP_BMAP:
                val = READ_ONCE(smccc_feat->std_hyp_bmap);
                break;
        case KVM_REG_ARM_VENDOR_HYP_BMAP:
                val = READ_ONCE(smccc_feat->vendor_hyp_bmap);
                break;
        case KVM_REG_ARM_VENDOR_HYP_BMAP_2:
                val = READ_ONCE(smccc_feat->vendor_hyp_bmap_2);
                break;
        default:
                return -ENOENT;
        }

        if (copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id)))
                return -EFAULT;

        return 0;
}

static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val)
{
        int ret = 0;
        struct kvm *kvm = vcpu->kvm;
        struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat;
        unsigned long *fw_reg_bmap, fw_reg_features;

        switch (reg_id) {
        case KVM_REG_ARM_STD_BMAP:
                fw_reg_bmap = &smccc_feat->std_bmap;
                fw_reg_features = KVM_ARM_SMCCC_STD_FEATURES;
                break;
        case KVM_REG_ARM_STD_HYP_BMAP:
                fw_reg_bmap = &smccc_feat->std_hyp_bmap;
                fw_reg_features = KVM_ARM_SMCCC_STD_HYP_FEATURES;
                break;
        case KVM_REG_ARM_VENDOR_HYP_BMAP:
                fw_reg_bmap = &smccc_feat->vendor_hyp_bmap;
                fw_reg_features = KVM_ARM_SMCCC_VENDOR_HYP_FEATURES;
                break;
        case KVM_REG_ARM_VENDOR_HYP_BMAP_2:
                fw_reg_bmap = &smccc_feat->vendor_hyp_bmap_2;
                fw_reg_features = KVM_ARM_SMCCC_VENDOR_HYP_FEATURES_2;
                break;
        default:
                return -ENOENT;
        }

        /* Check for unsupported bit */
        if (val & ~fw_reg_features)
                return -EINVAL;

        mutex_lock(&kvm->arch.config_lock);

        if (kvm_vm_has_ran_once(kvm) && val != *fw_reg_bmap) {
                ret = -EBUSY;
                goto out;
        }

        WRITE_ONCE(*fw_reg_bmap, val);
out:
        mutex_unlock(&kvm->arch.config_lock);
        return ret;
}

int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
{
        void __user *uaddr = (void __user *)(long)reg->addr;
        u64 val;
        int wa_level;

        if (KVM_REG_SIZE(reg->id) != sizeof(val))
                return -ENOENT;
        if (copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id)))
                return -EFAULT;

        switch (reg->id) {
        case KVM_REG_ARM_PSCI_VERSION:
        {
                bool wants_02;

                wants_02 = vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2);

                switch (val) {
                case KVM_ARM_PSCI_0_1:
                        if (wants_02)
                                return -EINVAL;
                        vcpu->kvm->arch.psci_version = val;
                        return 0;
                case KVM_ARM_PSCI_0_2:
                case KVM_ARM_PSCI_1_0:
                case KVM_ARM_PSCI_1_1:
                case KVM_ARM_PSCI_1_2:
                case KVM_ARM_PSCI_1_3:
                        if (!wants_02)
                                return -EINVAL;
                        vcpu->kvm->arch.psci_version = val;
                        return 0;
                }
                break;
        }

        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:
        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3:
                if (val & ~KVM_REG_FEATURE_LEVEL_MASK)
                        return -EINVAL;

                if (get_kernel_wa_level(vcpu, reg->id) < val)
                        return -EINVAL;

                return 0;

        case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2:
                if (val & ~(KVM_REG_FEATURE_LEVEL_MASK |
                            KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED))
                        return -EINVAL;

                /* The enabled bit must not be set unless the level is AVAIL. */
                if ((val & KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED) &&
                    (val & KVM_REG_FEATURE_LEVEL_MASK) != KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL)
                        return -EINVAL;

                /*
                 * Map all the possible incoming states to the only two we
                 * really want to deal with.
                 */
                switch (val & KVM_REG_FEATURE_LEVEL_MASK) {
                case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL:
                case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN:
                        wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL;
                        break;
                case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL:
                case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED:
                        wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED;
                        break;
                default:
                        return -EINVAL;
                }

                /*
                 * We can deal with NOT_AVAIL on NOT_REQUIRED, but not the
                 * other way around.
                 */
                if (get_kernel_wa_level(vcpu, reg->id) < wa_level)
                        return -EINVAL;

                return 0;
        case KVM_REG_ARM_STD_BMAP:
        case KVM_REG_ARM_STD_HYP_BMAP:
        case KVM_REG_ARM_VENDOR_HYP_BMAP:
        case KVM_REG_ARM_VENDOR_HYP_BMAP_2:
                return kvm_arm_set_fw_reg_bmap(vcpu, reg->id, val);
        default:
                return -ENOENT;
        }

        return -EINVAL;
}

int kvm_vm_smccc_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
{
        switch (attr->attr) {
        case KVM_ARM_VM_SMCCC_FILTER:
                return 0;
        default:
                return -ENXIO;
        }
}

int kvm_vm_smccc_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
{
        void __user *uaddr = (void __user *)attr->addr;

        switch (attr->attr) {
        case KVM_ARM_VM_SMCCC_FILTER:
                return kvm_smccc_set_filter(kvm, uaddr);
        default:
                return -ENXIO;
        }
}

























    5 

    2 

    1 

    1 

































    5 


    2 


    1 


    1 





























































































































    5 








    5 





    1 







    5 

    3 

    5 

    5 





    5 



    1 


    2 
    5 









    5 







  295 
















  181 

  115 




























  295 










  295 




  248 


   93 
  156 







  246 



   93 

  156 








  248 











   94 











   94 




















  116 











  116 

























































































  115 









   93 








   94 
   94 
    1 
   93 



  116 
  115 






    1 
  115 
  116 














































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2012-2015 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 */

#include <hyp/adjust_pc.h>

#include <linux/compiler.h>
#include <linux/irqchip/arm-gic-v3.h>
#include <linux/kvm_host.h>

#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>

#define vtr_to_max_lr_idx(v)                ((v) & 0xf)
#define vtr_to_nr_pre_bits(v)                ((((u32)(v) >> 26) & 7) + 1)
#define vtr_to_nr_apr_regs(v)                (1 << (vtr_to_nr_pre_bits(v) - 5))

u64 __gic_v3_get_lr(unsigned int lr)
{
        switch (lr & 0xf) {
        case 0:
                return read_gicreg(ICH_LR0_EL2);
        case 1:
                return read_gicreg(ICH_LR1_EL2);
        case 2:
                return read_gicreg(ICH_LR2_EL2);
        case 3:
                return read_gicreg(ICH_LR3_EL2);
        case 4:
                return read_gicreg(ICH_LR4_EL2);
        case 5:
                return read_gicreg(ICH_LR5_EL2);
        case 6:
                return read_gicreg(ICH_LR6_EL2);
        case 7:
                return read_gicreg(ICH_LR7_EL2);
        case 8:
                return read_gicreg(ICH_LR8_EL2);
        case 9:
                return read_gicreg(ICH_LR9_EL2);
        case 10:
                return read_gicreg(ICH_LR10_EL2);
        case 11:
                return read_gicreg(ICH_LR11_EL2);
        case 12:
                return read_gicreg(ICH_LR12_EL2);
        case 13:
                return read_gicreg(ICH_LR13_EL2);
        case 14:
                return read_gicreg(ICH_LR14_EL2);
        case 15:
                return read_gicreg(ICH_LR15_EL2);
        }

        unreachable();
}

static void __gic_v3_set_lr(u64 val, int lr)
{
        switch (lr & 0xf) {
        case 0:
                write_gicreg(val, ICH_LR0_EL2);
                break;
        case 1:
                write_gicreg(val, ICH_LR1_EL2);
                break;
        case 2:
                write_gicreg(val, ICH_LR2_EL2);
                break;
        case 3:
                write_gicreg(val, ICH_LR3_EL2);
                break;
        case 4:
                write_gicreg(val, ICH_LR4_EL2);
                break;
        case 5:
                write_gicreg(val, ICH_LR5_EL2);
                break;
        case 6:
                write_gicreg(val, ICH_LR6_EL2);
                break;
        case 7:
                write_gicreg(val, ICH_LR7_EL2);
                break;
        case 8:
                write_gicreg(val, ICH_LR8_EL2);
                break;
        case 9:
                write_gicreg(val, ICH_LR9_EL2);
                break;
        case 10:
                write_gicreg(val, ICH_LR10_EL2);
                break;
        case 11:
                write_gicreg(val, ICH_LR11_EL2);
                break;
        case 12:
                write_gicreg(val, ICH_LR12_EL2);
                break;
        case 13:
                write_gicreg(val, ICH_LR13_EL2);
                break;
        case 14:
                write_gicreg(val, ICH_LR14_EL2);
                break;
        case 15:
                write_gicreg(val, ICH_LR15_EL2);
                break;
        }
}

static void __vgic_v3_write_ap0rn(u32 val, int n)
{
        switch (n) {
        case 0:
                write_gicreg(val, ICH_AP0R0_EL2);
                break;
        case 1:
                write_gicreg(val, ICH_AP0R1_EL2);
                break;
        case 2:
                write_gicreg(val, ICH_AP0R2_EL2);
                break;
        case 3:
                write_gicreg(val, ICH_AP0R3_EL2);
                break;
        }
}

static void __vgic_v3_write_ap1rn(u32 val, int n)
{
        switch (n) {
        case 0:
                write_gicreg(val, ICH_AP1R0_EL2);
                break;
        case 1:
                write_gicreg(val, ICH_AP1R1_EL2);
                break;
        case 2:
                write_gicreg(val, ICH_AP1R2_EL2);
                break;
        case 3:
                write_gicreg(val, ICH_AP1R3_EL2);
                break;
        }
}

static u32 __vgic_v3_read_ap0rn(int n)
{
        u32 val;

        switch (n) {
        case 0:
                val = read_gicreg(ICH_AP0R0_EL2);
                break;
        case 1:
                val = read_gicreg(ICH_AP0R1_EL2);
                break;
        case 2:
                val = read_gicreg(ICH_AP0R2_EL2);
                break;
        case 3:
                val = read_gicreg(ICH_AP0R3_EL2);
                break;
        default:
                unreachable();
        }

        return val;
}

static u32 __vgic_v3_read_ap1rn(int n)
{
        u32 val;

        switch (n) {
        case 0:
                val = read_gicreg(ICH_AP1R0_EL2);
                break;
        case 1:
                val = read_gicreg(ICH_AP1R1_EL2);
                break;
        case 2:
                val = read_gicreg(ICH_AP1R2_EL2);
                break;
        case 3:
                val = read_gicreg(ICH_AP1R3_EL2);
                break;
        default:
                unreachable();
        }

        return val;
}

void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
{
        u64 used_lrs = cpu_if->used_lrs;

        /*
         * Make sure stores to the GIC via the memory mapped interface
         * are now visible to the system register interface when reading the
         * LRs, and when reading back the VMCR on non-VHE systems.
         */
        if (used_lrs || !has_vhe()) {
                if (!cpu_if->vgic_sre) {
                        dsb(sy);
                        isb();
                }
        }

        if (used_lrs || cpu_if->its_vpe.its_vm) {
                int i;
                u32 elrsr;

                elrsr = read_gicreg(ICH_ELRSR_EL2);

                write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2);

                for (i = 0; i < used_lrs; i++) {
                        if (elrsr & (1 << i))
                                cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
                        else
                                cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);

                        __gic_v3_set_lr(0, i);
                }
        }
}

void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
{
        u64 used_lrs = cpu_if->used_lrs;
        int i;

        if (used_lrs || cpu_if->its_vpe.its_vm) {
                write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);

                for (i = 0; i < used_lrs; i++)
                        __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
        }

        /*
         * Ensure that writes to the LRs, and on non-VHE systems ensure that
         * the write to the VMCR in __vgic_v3_activate_traps(), will have
         * reached the (re)distributors. This ensure the guest will read the
         * correct values from the memory-mapped interface.
         */
        if (used_lrs || !has_vhe()) {
                if (!cpu_if->vgic_sre) {
                        isb();
                        dsb(sy);
                }
        }
}

void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
{
        /*
         * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
         * Group0 interrupt (as generated in GICv2 mode) to be
         * delivered as a FIQ to the guest, with potentially fatal
         * consequences. So we must make sure that ICC_SRE_EL1 has
         * been actually programmed with the value we want before
         * starting to mess with the rest of the GIC, and VMCR_EL2 in
         * particular.  This logic must be called before
         * __vgic_v3_restore_state().
         *
         * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
         * provisioned at all. In order to prevent illegal accesses to the
         * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
         * so that the trap bits can take effect. Yes, we *loves* the GIC.
         */
        if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) {
                write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
                isb();
        } else if (!cpu_if->vgic_sre) {
                write_gicreg(0, ICC_SRE_EL1);
                isb();
                write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);


                if (has_vhe()) {
                        /*
                         * Ensure that the write to the VMCR will have reached
                         * the (re)distributors. This ensure the guest will
                         * read the correct values from the memory-mapped
                         * interface.
                         */
                        isb();
                        dsb(sy);
                }
        }

        /*
         * GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due
         * to be relaxed in a future spec release, at which point this in
         * condition can be dropped.
         */
        if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
                /*
                 * Prevent the guest from touching the ICC_SRE_EL1 system
                 * register. Note that this may not have any effect, as
                 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
                 */
                write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
                             ICC_SRE_EL2);
        }

        /*
         * If we need to trap system registers, we must write
         * ICH_HCR_EL2 anyway, even if no interrupts are being
         * injected. Note that this also applies if we don't expect
         * any system register access (no vgic at all).
         */
        if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
            cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
                write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
}

void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
{
        u64 val;

        if (!cpu_if->vgic_sre) {
                cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
        }

        /*
         * Can be dropped in the future when GICv5 spec is relaxed. See comment
         * above.
         */
        if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
                val = read_gicreg(ICC_SRE_EL2);
                write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
        }

        if (!cpu_if->vgic_sre) {
                /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
                isb();
                write_gicreg(1, ICC_SRE_EL1);
        }

        /*
         * If we were trapping system registers, we enabled the VGIC even if
         * no interrupts were being injected, and we disable it again here.
         */
        if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
            cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
                write_gicreg(0, ICH_HCR_EL2);
}

static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
{
        u64 val;
        u32 nr_pre_bits;

        val = read_gicreg(ICH_VTR_EL2);
        nr_pre_bits = vtr_to_nr_pre_bits(val);

        switch (nr_pre_bits) {
        case 7:
                cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
                cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
                fallthrough;
        case 6:
                cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
                fallthrough;
        default:
                cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
        }

        switch (nr_pre_bits) {
        case 7:
                cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
                cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
                fallthrough;
        case 6:
                cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
                fallthrough;
        default:
                cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
        }
}

static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
{
        u64 val;
        u32 nr_pre_bits;

        val = read_gicreg(ICH_VTR_EL2);
        nr_pre_bits = vtr_to_nr_pre_bits(val);

        switch (nr_pre_bits) {
        case 7:
                __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
                __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
                fallthrough;
        case 6:
                __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
                fallthrough;
        default:
                __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
        }

        switch (nr_pre_bits) {
        case 7:
                __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
                __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
                fallthrough;
        case 6:
                __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
                fallthrough;
        default:
                __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
        }
}

void __vgic_v3_init_lrs(void)
{
        int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
        int i;

        for (i = 0; i <= max_lr_idx; i++)
                __gic_v3_set_lr(0, i);
}

/*
 * Return the GIC CPU configuration:
 * - [31:0]  ICH_VTR_EL2
 * - [62:32] RES0
 * - [63]    MMIO (GICv2) capable
 */
u64 __vgic_v3_get_gic_config(void)
{
        u64 val, sre;
        unsigned long flags = 0;

        /*
         * In compat mode, we cannot access ICC_SRE_EL1 at any EL
         * other than EL1 itself; just return the
         * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
         * system, so we first check if we have GICv5 support.
         */
        if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
                return read_gicreg(ICH_VTR_EL2);

        sre = read_gicreg(ICC_SRE_EL1);
        /*
         * To check whether we have a MMIO-based (GICv2 compatible)
         * CPU interface, we need to disable the system register
         * view.
         *
         * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
         * that to be able to set ICC_SRE_EL1.SRE to 0, all the
         * interrupt overrides must be set. You've got to love this.
         *
         * As we always run VHE with HCR_xMO set, no extra xMO
         * manipulation is required in that case.
         *
         * To safely disable SRE, we have to prevent any interrupt
         * from firing (which would be deadly). This only makes sense
         * on VHE, as interrupts are already masked for nVHE as part
         * of the exception entry to EL2.
         */
        if (has_vhe()) {
                flags = local_daif_save();
        } else {
                sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
                isb();
        }

        write_gicreg(0, ICC_SRE_EL1);
        isb();

        val = read_gicreg(ICC_SRE_EL1);

        write_gicreg(sre, ICC_SRE_EL1);
        isb();

        if (has_vhe()) {
                local_daif_restore(flags);
        } else {
                sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
                isb();
        }

        val  = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
        val |= read_gicreg(ICH_VTR_EL2);

        return val;
}

static void __vgic_v3_compat_mode_enable(void)
{
        if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
                return;

        sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
        /* Wait for V3 to become enabled */
        isb();
}

static u64 __vgic_v3_read_vmcr(void)
{
        return read_gicreg(ICH_VMCR_EL2);
}

static void __vgic_v3_write_vmcr(u32 vmcr)
{
        write_gicreg(vmcr, ICH_VMCR_EL2);
}

void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
{
        __vgic_v3_save_aprs(cpu_if);
        if (cpu_if->vgic_sre)
                cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
}

void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
{
        __vgic_v3_compat_mode_enable();

        /*
         * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
         * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
         * VMCR_EL2 save/restore in the world switch.
         */
        if (cpu_if->vgic_sre)
                __vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
        __vgic_v3_restore_aprs(cpu_if);
}

static int __vgic_v3_bpr_min(void)
{
        /* See Pseudocode for VPriorityGroup */
        return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
}

static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
{
        u64 esr = kvm_vcpu_get_esr(vcpu);
        u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;

        return crm != 8;
}

#define GICv3_IDLE_PRIORITY        0xff

static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
                                         u64 *lr_val)
{
        unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
        u8 priority = GICv3_IDLE_PRIORITY;
        int i, lr = -1;

        for (i = 0; i < used_lrs; i++) {
                u64 val = __gic_v3_get_lr(i);
                u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;

                /* Not pending in the state? */
                if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
                        continue;

                /* Group-0 interrupt, but Group-0 disabled? */
                if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
                        continue;

                /* Group-1 interrupt, but Group-1 disabled? */
                if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
                        continue;

                /* Not the highest priority? */
                if (lr_prio >= priority)
                        continue;

                /* This is a candidate */
                priority = lr_prio;
                *lr_val = val;
                lr = i;
        }

        if (lr == -1)
                *lr_val = ICC_IAR1_EL1_SPURIOUS;

        return lr;
}

static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
                                    u64 *lr_val)
{
        unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
        int i;

        for (i = 0; i < used_lrs; i++) {
                u64 val = __gic_v3_get_lr(i);

                if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
                    (val & ICH_LR_ACTIVE_BIT)) {
                        *lr_val = val;
                        return i;
                }
        }

        *lr_val = ICC_IAR1_EL1_SPURIOUS;
        return -1;
}

static int __vgic_v3_get_highest_active_priority(void)
{
        u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
        u32 hap = 0;
        int i;

        for (i = 0; i < nr_apr_regs; i++) {
                u32 val;

                /*
                 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
                 * contain the active priority levels for this VCPU
                 * for the maximum number of supported priority
                 * levels, and we return the full priority level only
                 * if the BPR is programmed to its minimum, otherwise
                 * we return a combination of the priority level and
                 * subpriority, as determined by the setting of the
                 * BPR, but without the full subpriority.
                 */
                val  = __vgic_v3_read_ap0rn(i);
                val |= __vgic_v3_read_ap1rn(i);
                if (!val) {
                        hap += 32;
                        continue;
                }

                return (hap + __ffs(val)) << __vgic_v3_bpr_min();
        }

        return GICv3_IDLE_PRIORITY;
}

static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
{
        return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
}

static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
{
        unsigned int bpr;

        if (vmcr & ICH_VMCR_CBPR_MASK) {
                bpr = __vgic_v3_get_bpr0(vmcr);
                if (bpr < 7)
                        bpr++;
        } else {
                bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
        }

        return bpr;
}

/*
 * Convert a priority to a preemption level, taking the relevant BPR
 * into account by zeroing the sub-priority bits.
 */
static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
{
        unsigned int bpr;

        if (!grp)
                bpr = __vgic_v3_get_bpr0(vmcr) + 1;
        else
                bpr = __vgic_v3_get_bpr1(vmcr);

        return pri & (GENMASK(7, 0) << bpr);
}

/*
 * The priority value is independent of any of the BPR values, so we
 * normalize it using the minimal BPR value. This guarantees that no
 * matter what the guest does with its BPR, we can always set/get the
 * same value of a priority.
 */
static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
{
        u8 pre, ap;
        u32 val;
        int apr;

        pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
        ap = pre >> __vgic_v3_bpr_min();
        apr = ap / 32;

        if (!grp) {
                val = __vgic_v3_read_ap0rn(apr);
                __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
        } else {
                val = __vgic_v3_read_ap1rn(apr);
                __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
        }
}

static int __vgic_v3_clear_highest_active_priority(void)
{
        u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
        u32 hap = 0;
        int i;

        for (i = 0; i < nr_apr_regs; i++) {
                u32 ap0, ap1;
                int c0, c1;

                ap0 = __vgic_v3_read_ap0rn(i);
                ap1 = __vgic_v3_read_ap1rn(i);
                if (!ap0 && !ap1) {
                        hap += 32;
                        continue;
                }

                c0 = ap0 ? __ffs(ap0) : 32;
                c1 = ap1 ? __ffs(ap1) : 32;

                /* Always clear the LSB, which is the highest priority */
                if (c0 < c1) {
                        ap0 &= ~BIT(c0);
                        __vgic_v3_write_ap0rn(ap0, i);
                        hap += c0;
                } else {
                        ap1 &= ~BIT(c1);
                        __vgic_v3_write_ap1rn(ap1, i);
                        hap += c1;
                }

                /* Rescale to 8 bits of priority */
                return hap << __vgic_v3_bpr_min();
        }

        return GICv3_IDLE_PRIORITY;
}

static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 lr_val;
        u8 lr_prio, pmr;
        int lr, grp;

        grp = __vgic_v3_get_group(vcpu);

        lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
        if (lr < 0)
                goto spurious;

        if (grp != !!(lr_val & ICH_LR_GROUP))
                goto spurious;

        pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
        lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
        if (pmr <= lr_prio)
                goto spurious;

        if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
                goto spurious;

        lr_val &= ~ICH_LR_STATE;
        lr_val |= ICH_LR_ACTIVE_BIT;
        __gic_v3_set_lr(lr_val, lr);
        __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
        vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
        return;

spurious:
        vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
}

static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
{
        lr_val &= ~ICH_LR_ACTIVE_BIT;
        if (lr_val & ICH_LR_HW) {
                u32 pid;

                pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
                gic_write_dir(pid);
        }

        __gic_v3_set_lr(lr_val, lr);
}

static void __vgic_v3_bump_eoicount(void)
{
        u32 hcr;

        hcr = read_gicreg(ICH_HCR_EL2);
        hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT;
        write_gicreg(hcr, ICH_HCR_EL2);
}

static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 vid = vcpu_get_reg(vcpu, rt);
        u64 lr_val;
        int lr;

        /* EOImode == 0, nothing to be done here */
        if (!(vmcr & ICH_VMCR_EOIM_MASK))
                return;

        /* No deactivate to be performed on an LPI */
        if (vid >= VGIC_MIN_LPI)
                return;

        lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
        if (lr == -1) {
                __vgic_v3_bump_eoicount();
                return;
        }

        __vgic_v3_clear_active_lr(lr, lr_val);
}

static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 vid = vcpu_get_reg(vcpu, rt);
        u64 lr_val;
        u8 lr_prio, act_prio;
        int lr, grp;

        grp = __vgic_v3_get_group(vcpu);

        /* Drop priority in any case */
        act_prio = __vgic_v3_clear_highest_active_priority();

        lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
        if (lr == -1) {
                /* Do not bump EOIcount for LPIs that aren't in the LRs */
                if (!(vid >= VGIC_MIN_LPI))
                        __vgic_v3_bump_eoicount();
                return;
        }

        /* EOImode == 1 and not an LPI, nothing to be done here */
        if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
                return;

        lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;

        /* If priorities or group do not match, the guest has fscked-up. */
        if (grp != !!(lr_val & ICH_LR_GROUP) ||
            __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
                return;

        /* Let's now perform the deactivation */
        __vgic_v3_clear_active_lr(lr, lr_val);
}

static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
}

static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
}

static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 val = vcpu_get_reg(vcpu, rt);

        if (val & 1)
                vmcr |= ICH_VMCR_ENG0_MASK;
        else
                vmcr &= ~ICH_VMCR_ENG0_MASK;

        __vgic_v3_write_vmcr(vmcr);
}

static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 val = vcpu_get_reg(vcpu, rt);

        if (val & 1)
                vmcr |= ICH_VMCR_ENG1_MASK;
        else
                vmcr &= ~ICH_VMCR_ENG1_MASK;

        __vgic_v3_write_vmcr(vmcr);
}

static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
}

static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
}

static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 val = vcpu_get_reg(vcpu, rt);
        u8 bpr_min = __vgic_v3_bpr_min() - 1;

        /* Enforce BPR limiting */
        if (val < bpr_min)
                val = bpr_min;

        val <<= ICH_VMCR_BPR0_SHIFT;
        val &= ICH_VMCR_BPR0_MASK;
        vmcr &= ~ICH_VMCR_BPR0_MASK;
        vmcr |= val;

        __vgic_v3_write_vmcr(vmcr);
}

static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 val = vcpu_get_reg(vcpu, rt);
        u8 bpr_min = __vgic_v3_bpr_min();

        if (vmcr & ICH_VMCR_CBPR_MASK)
                return;

        /* Enforce BPR limiting */
        if (val < bpr_min)
                val = bpr_min;

        val <<= ICH_VMCR_BPR1_SHIFT;
        val &= ICH_VMCR_BPR1_MASK;
        vmcr &= ~ICH_VMCR_BPR1_MASK;
        vmcr |= val;

        __vgic_v3_write_vmcr(vmcr);
}

static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
{
        u32 val;

        if (!__vgic_v3_get_group(vcpu))
                val = __vgic_v3_read_ap0rn(n);
        else
                val = __vgic_v3_read_ap1rn(n);

        vcpu_set_reg(vcpu, rt, val);
}

static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
{
        u32 val = vcpu_get_reg(vcpu, rt);

        if (!__vgic_v3_get_group(vcpu))
                __vgic_v3_write_ap0rn(val, n);
        else
                __vgic_v3_write_ap1rn(val, n);
}

static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
                                            u32 vmcr, int rt)
{
        __vgic_v3_read_apxrn(vcpu, rt, 0);
}

static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
                                            u32 vmcr, int rt)
{
        __vgic_v3_read_apxrn(vcpu, rt, 1);
}

static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_read_apxrn(vcpu, rt, 2);
}

static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_read_apxrn(vcpu, rt, 3);
}

static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_write_apxrn(vcpu, rt, 0);
}

static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_write_apxrn(vcpu, rt, 1);
}

static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_write_apxrn(vcpu, rt, 2);
}

static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        __vgic_v3_write_apxrn(vcpu, rt, 3);
}

static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u64 lr_val;
        int lr, lr_grp, grp;

        grp = __vgic_v3_get_group(vcpu);

        lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
        if (lr == -1)
                goto spurious;

        lr_grp = !!(lr_val & ICH_LR_GROUP);
        if (lr_grp != grp)
                lr_val = ICC_IAR1_EL1_SPURIOUS;

spurious:
        vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
}

static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        vmcr &= ICH_VMCR_PMR_MASK;
        vmcr >>= ICH_VMCR_PMR_SHIFT;
        vcpu_set_reg(vcpu, rt, vmcr);
}

static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 val = vcpu_get_reg(vcpu, rt);

        val <<= ICH_VMCR_PMR_SHIFT;
        val &= ICH_VMCR_PMR_MASK;
        vmcr &= ~ICH_VMCR_PMR_MASK;
        vmcr |= val;

        write_gicreg(vmcr, ICH_VMCR_EL2);
}

static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 val = __vgic_v3_get_highest_active_priority();
        vcpu_set_reg(vcpu, rt, val);
}

static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 vtr, val;

        vtr = read_gicreg(ICH_VTR_EL2);
        /* PRIbits */
        val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
        /* IDbits */
        val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
        /* A3V */
        val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
        /* EOImode */
        val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
        /* CBPR */
        val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;

        vcpu_set_reg(vcpu, rt, val);
}

static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
{
        u32 val = vcpu_get_reg(vcpu, rt);

        if (val & ICC_CTLR_EL1_CBPR_MASK)
                vmcr |= ICH_VMCR_CBPR_MASK;
        else
                vmcr &= ~ICH_VMCR_CBPR_MASK;

        if (val & ICC_CTLR_EL1_EOImode_MASK)
                vmcr |= ICH_VMCR_EOIM_MASK;
        else
                vmcr &= ~ICH_VMCR_EOIM_MASK;

        write_gicreg(vmcr, ICH_VMCR_EL2);
}

static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
                                            u32 sysreg, bool is_read)
{
        u64 ich_hcr;

        if (!is_nested_ctxt(vcpu))
                return false;

        ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);

        switch (sysreg) {
        case SYS_ICC_IGRPEN0_EL1:
                if (is_read &&
                    (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
                        return true;

                if (!is_read &&
                    (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
                        return true;

                fallthrough;

        case SYS_ICC_AP0Rn_EL1(0):
        case SYS_ICC_AP0Rn_EL1(1):
        case SYS_ICC_AP0Rn_EL1(2):
        case SYS_ICC_AP0Rn_EL1(3):
        case SYS_ICC_BPR0_EL1:
        case SYS_ICC_EOIR0_EL1:
        case SYS_ICC_HPPIR0_EL1:
        case SYS_ICC_IAR0_EL1:
                return ich_hcr & ICH_HCR_EL2_TALL0;

        case SYS_ICC_IGRPEN1_EL1:
                if (is_read &&
                    (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
                        return true;

                if (!is_read &&
                    (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
                        return true;

                fallthrough;

        case SYS_ICC_AP1Rn_EL1(0):
        case SYS_ICC_AP1Rn_EL1(1):
        case SYS_ICC_AP1Rn_EL1(2):
        case SYS_ICC_AP1Rn_EL1(3):
        case SYS_ICC_BPR1_EL1:
        case SYS_ICC_EOIR1_EL1:
        case SYS_ICC_HPPIR1_EL1:
        case SYS_ICC_IAR1_EL1:
                return ich_hcr & ICH_HCR_EL2_TALL1;

        case SYS_ICC_DIR_EL1:
                if (ich_hcr & ICH_HCR_EL2_TDIR)
                        return true;

                fallthrough;

        case SYS_ICC_RPR_EL1:
        case SYS_ICC_CTLR_EL1:
        case SYS_ICC_PMR_EL1:
                return ich_hcr & ICH_HCR_EL2_TC;

        default:
                return false;
        }
}

int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
{
        int rt;
        u64 esr;
        u32 vmcr;
        void (*fn)(struct kvm_vcpu *, u32, int);
        bool is_read;
        u32 sysreg;

        if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
                return 0;

        esr = kvm_vcpu_get_esr(vcpu);
        if (vcpu_mode_is_32bit(vcpu)) {
                if (!kvm_condition_valid(vcpu)) {
                        __kvm_skip_instr(vcpu);
                        return 1;
                }

                sysreg = esr_cp15_to_sysreg(esr);
        } else {
                sysreg = esr_sys64_to_sysreg(esr);
        }

        is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;

        if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
                return 0;

        switch (sysreg) {
        case SYS_ICC_IAR0_EL1:
        case SYS_ICC_IAR1_EL1:
                if (unlikely(!is_read))
                        return 0;
                fn = __vgic_v3_read_iar;
                break;
        case SYS_ICC_EOIR0_EL1:
        case SYS_ICC_EOIR1_EL1:
                if (unlikely(is_read))
                        return 0;
                fn = __vgic_v3_write_eoir;
                break;
        case SYS_ICC_IGRPEN1_EL1:
                if (is_read)
                        fn = __vgic_v3_read_igrpen1;
                else
                        fn = __vgic_v3_write_igrpen1;
                break;
        case SYS_ICC_BPR1_EL1:
                if (is_read)
                        fn = __vgic_v3_read_bpr1;
                else
                        fn = __vgic_v3_write_bpr1;
                break;
        case SYS_ICC_AP0Rn_EL1(0):
        case SYS_ICC_AP1Rn_EL1(0):
                if (is_read)
                        fn = __vgic_v3_read_apxr0;
                else
                        fn = __vgic_v3_write_apxr0;
                break;
        case SYS_ICC_AP0Rn_EL1(1):
        case SYS_ICC_AP1Rn_EL1(1):
                if (is_read)
                        fn = __vgic_v3_read_apxr1;
                else
                        fn = __vgic_v3_write_apxr1;
                break;
        case SYS_ICC_AP0Rn_EL1(2):
        case SYS_ICC_AP1Rn_EL1(2):
                if (is_read)
                        fn = __vgic_v3_read_apxr2;
                else
                        fn = __vgic_v3_write_apxr2;
                break;
        case SYS_ICC_AP0Rn_EL1(3):
        case SYS_ICC_AP1Rn_EL1(3):
                if (is_read)
                        fn = __vgic_v3_read_apxr3;
                else
                        fn = __vgic_v3_write_apxr3;
                break;
        case SYS_ICC_HPPIR0_EL1:
        case SYS_ICC_HPPIR1_EL1:
                if (unlikely(!is_read))
                        return 0;
                fn = __vgic_v3_read_hppir;
                break;
        case SYS_ICC_IGRPEN0_EL1:
                if (is_read)
                        fn = __vgic_v3_read_igrpen0;
                else
                        fn = __vgic_v3_write_igrpen0;
                break;
        case SYS_ICC_BPR0_EL1:
                if (is_read)
                        fn = __vgic_v3_read_bpr0;
                else
                        fn = __vgic_v3_write_bpr0;
                break;
        case SYS_ICC_DIR_EL1:
                if (unlikely(is_read))
                        return 0;
                fn = __vgic_v3_write_dir;
                break;
        case SYS_ICC_RPR_EL1:
                if (unlikely(!is_read))
                        return 0;
                fn = __vgic_v3_read_rpr;
                break;
        case SYS_ICC_CTLR_EL1:
                if (is_read)
                        fn = __vgic_v3_read_ctlr;
                else
                        fn = __vgic_v3_write_ctlr;
                break;
        case SYS_ICC_PMR_EL1:
                if (is_read)
                        fn = __vgic_v3_read_pmr;
                else
                        fn = __vgic_v3_write_pmr;
                break;
        default:
                return 0;
        }

        vmcr = __vgic_v3_read_vmcr();
        rt = kvm_vcpu_sys_get_rt(vcpu);
        fn(vcpu, vmcr, rt);

        __kvm_skip_instr(vcpu);

        return 1;
}














  292 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_GENERIC_BITOPS_BUILTIN_FLS_H_
#define _ASM_GENERIC_BITOPS_BUILTIN_FLS_H_

/**
 * fls - find last (most-significant) bit set
 * @x: the word to search
 *
 * This is defined the same way as ffs.
 * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 */
static __always_inline int fls(unsigned int x)
{
        return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
}

#endif


































































































































































































    1 












    1 



























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_WAIT_H
#define _LINUX_WAIT_H
/*
 * Linux wait queue related types and methods
 */
#include <linux/list.h>
#include <linux/stddef.h>
#include <linux/spinlock.h>

#include <asm/current.h>

typedef struct wait_queue_entry wait_queue_entry_t;

typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);

/* wait_queue_entry::flags */
#define WQ_FLAG_EXCLUSIVE        0x01
#define WQ_FLAG_WOKEN                0x02
#define WQ_FLAG_CUSTOM                0x04
#define WQ_FLAG_DONE                0x08
#define WQ_FLAG_PRIORITY        0x10

/*
 * A single wait-queue entry structure:
 */
struct wait_queue_entry {
        unsigned int                flags;
        void                        *private;
        wait_queue_func_t        func;
        struct list_head        entry;
};

struct wait_queue_head {
        spinlock_t                lock;
        struct list_head        head;
};
typedef struct wait_queue_head wait_queue_head_t;

struct task_struct;

/*
 * Macros for declaration and initialisaton of the datatypes
 */

#define __WAITQUEUE_INITIALIZER(name, tsk) {                                        \
        .private        = tsk,                                                        \
        .func                = default_wake_function,                                \
        .entry                = { NULL, NULL } }

#define DECLARE_WAITQUEUE(name, tsk)                                                \
        struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)

#define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                        \
        .lock                = __SPIN_LOCK_UNLOCKED(name.lock),                        \
        .head                = LIST_HEAD_INIT(name.head) }

#define DECLARE_WAIT_QUEUE_HEAD(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)

extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);

#define init_waitqueue_head(wq_head)                                                \
        do {                                                                        \
                static struct lock_class_key __key;                                \
                                                                                \
                __init_waitqueue_head((wq_head), #wq_head, &__key);                \
        } while (0)

#ifdef CONFIG_LOCKDEP
# define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
        ({ init_waitqueue_head(&name); name; })
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
        struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
#else
# define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
#endif

static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
{
        wq_entry->flags                = 0;
        wq_entry->private        = p;
        wq_entry->func                = default_wake_function;
}

static inline void
init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
{
        wq_entry->flags                = 0;
        wq_entry->private        = NULL;
        wq_entry->func                = func;
}

/**
 * waitqueue_active -- locklessly test for waiters on the queue
 * @wq_head: the waitqueue to test for waiters
 *
 * returns true if the wait list is not empty
 *
 * NOTE: this function is lockless and requires care, incorrect usage _will_
 * lead to sporadic and non-obvious failure.
 *
 * Use either while holding wait_queue_head::lock or when used for wakeups
 * with an extra smp_mb() like::
 *
 *      CPU0 - waker                    CPU1 - waiter
 *
 *                                      for (;;) {
 *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 *      smp_mb();                         // smp_mb() from set_current_state()
 *      if (waitqueue_active(wq_head))         if (@cond)
 *        wake_up(wq_head);                      break;
 *                                        schedule();
 *                                      }
 *                                      finish_wait(&wq_head, &wait);
 *
 * Because without the explicit smp_mb() it's possible for the
 * waitqueue_active() load to get hoisted over the @cond store such that we'll
 * observe an empty wait list while the waiter might not observe @cond.
 *
 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 * which (when the lock is uncontended) are of roughly equal cost.
 */
static inline int waitqueue_active(struct wait_queue_head *wq_head)
{
        return !list_empty(&wq_head->head);
}

/**
 * wq_has_single_sleeper - check if there is only one sleeper
 * @wq_head: wait queue head
 *
 * Returns true of wq_head has only one sleeper on the list.
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
{
        return list_is_singular(&wq_head->head);
}

/**
 * wq_has_sleeper - check if there are any waiting processes
 * @wq_head: wait queue head
 *
 * Returns true if wq_head has waiting processes
 *
 * Please refer to the comment for waitqueue_active.
 */
static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
{
        /*
         * We need to be sure we are in sync with the
         * add_wait_queue modifications to the wait queue.
         *
         * This memory barrier should be paired with one on the
         * waiting side.
         */
        smp_mb();
        return waitqueue_active(wq_head);
}

extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);

static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        struct list_head *head = &wq_head->head;
        struct wait_queue_entry *wq;

        list_for_each_entry(wq, &wq_head->head, entry) {
                if (!(wq->flags & WQ_FLAG_PRIORITY))
                        break;
                head = &wq->entry;
        }
        list_add(&wq_entry->entry, head);
}

/*
 * Used for wake-one threads:
 */
static inline void
__add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue(wq_head, wq_entry);
}

static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_add_tail(&wq_entry->entry, &wq_head->head);
}

static inline void
__add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
        __add_wait_queue_entry_tail(wq_head, wq_entry);
}

static inline void
__remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
{
        list_del(&wq_entry->entry);
}

int __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
void __wake_up_on_current_cpu(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
void __wake_up_pollfree(struct wait_queue_head *wq_head);

#define wake_up(x)                        __wake_up(x, TASK_NORMAL, 1, NULL)
#define wake_up_nr(x, nr)                __wake_up(x, TASK_NORMAL, nr, NULL)
#define wake_up_all(x)                        __wake_up(x, TASK_NORMAL, 0, NULL)
#define wake_up_locked(x)                __wake_up_locked((x), TASK_NORMAL, 1)
#define wake_up_all_locked(x)                __wake_up_locked((x), TASK_NORMAL, 0)
#define wake_up_sync(x)                        __wake_up_sync(x, TASK_NORMAL)

#define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
#define wake_up_interruptible_nr(x, nr)        __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
#define wake_up_interruptible_all(x)        __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
#define wake_up_interruptible_sync(x)        __wake_up_sync((x), TASK_INTERRUPTIBLE)

/*
 * Wakeup macros to be used to report events to the targets.
 */
#define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
#define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
#define wake_up_poll(x, m)                                                        \
        __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
#define wake_up_poll_on_current_cpu(x, m)                                        \
        __wake_up_on_current_cpu(x, TASK_NORMAL, poll_to_key(m))
#define wake_up_locked_poll(x, m)                                                \
        __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
#define wake_up_interruptible_poll(x, m)                                        \
        __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
#define wake_up_interruptible_sync_poll(x, m)                                        \
        __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
#define wake_up_interruptible_sync_poll_locked(x, m)                                \
        __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))

/**
 * wake_up_pollfree - signal that a polled waitqueue is going away
 * @wq_head: the wait queue head
 *
 * In the very rare cases where a ->poll() implementation uses a waitqueue whose
 * lifetime is tied to a task rather than to the 'struct file' being polled,
 * this function must be called before the waitqueue is freed so that
 * non-blocking polls (e.g. epoll) are notified that the queue is going away.
 *
 * The caller must also RCU-delay the freeing of the wait_queue_head, e.g. via
 * an explicit synchronize_rcu() or call_rcu(), or via SLAB_TYPESAFE_BY_RCU.
 */
static inline void wake_up_pollfree(struct wait_queue_head *wq_head)
{
        /*
         * For performance reasons, we don't always take the queue lock here.
         * Therefore, we might race with someone removing the last entry from
         * the queue, and proceed while they still hold the queue lock.
         * However, rcu_read_lock() is required to be held in such cases, so we
         * can safely proceed with an RCU-delayed free.
         */
        if (waitqueue_active(wq_head))
                __wake_up_pollfree(wq_head);
}

#define ___wait_cond_timeout(condition)                                                \
({                                                                                \
        bool __cond = (condition);                                                \
        if (__cond && !__ret)                                                        \
                __ret = 1;                                                        \
        __cond || !__ret;                                                        \
})

#define ___wait_is_interruptible(state)                                                \
        (!__builtin_constant_p(state) ||                                        \
         (state & (TASK_INTERRUPTIBLE | TASK_WAKEKILL)))

extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);

/*
 * The below macro ___wait_event() has an explicit shadow of the __ret
 * variable when used from the wait_event_*() macros.
 *
 * This is so that both can use the ___wait_cond_timeout() construct
 * to wrap the condition.
 *
 * The type inconsistency of the wait_event_*() __ret variable is also
 * on purpose; we use long where we can return timeout values and int
 * otherwise.
 */

#define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)                \
({                                                                                \
        __label__ __out;                                                        \
        struct wait_queue_entry __wq_entry;                                        \
        long __ret = ret;        /* explicit shadow */                                \
                                                                                \
        init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
        for (;;) {                                                                \
                long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
                                                                                \
                if (condition)                                                        \
                        break;                                                        \
                                                                                \
                if (___wait_is_interruptible(state) && __int) {                        \
                        __ret = __int;                                                \
                        goto __out;                                                \
                }                                                                \
                                                                                \
                cmd;                                                                \
                                                                                \
                if (condition)                                                        \
                        break;                                                        \
        }                                                                        \
        finish_wait(&wq_head, &__wq_entry);                                        \
__out:        __ret;                                                                        \
})

#define __wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            schedule())

/**
 * wait_event - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event(wq_head, condition)                                                \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event(wq_head, condition);                                        \
} while (0)

#define __io_wait_event(wq_head, condition)                                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            io_schedule())

/*
 * io_wait_event() -- like wait_event() but with io_schedule()
 */
#define io_wait_event(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (condition)                                                                \
                break;                                                                \
        __io_wait_event(wq_head, condition);                                        \
} while (0)

#define __wait_event_freezable(wq_head, condition)                                \
        ___wait_event(wq_head, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE),        \
                        0, 0, schedule())

/**
 * wait_event_freezable - sleep (or freeze) until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 * to system load) until the @condition evaluates to true. The
 * @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_freezable(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_UNINTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_timeout(wq_head, condition, timeout)                                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_freezable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 0, timeout,                \
                      __ret = schedule_timeout(__ret))

/*
 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 * increasing load and is freezable.
 */
#define wait_event_freezable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
        __ret;                                                                        \
})

#define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,        \
                            cmd1; schedule(); cmd2)
/*
 * Just like wait_event_cmd(), except it sets exclusive flag
 */
#define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);                \
} while (0)

#define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            cmd1; schedule(); cmd2)

/**
 * wait_event_cmd - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @cmd1: the command will be executed before sleep
 * @cmd2: the command will be executed after sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 */
#define wait_event_cmd(wq_head, condition, cmd1, cmd2)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_cmd(wq_head, condition, cmd1, cmd2);                        \
} while (0)

#define __wait_event_interruptible(wq_head, condition)                                \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      schedule())

/**
 * wait_event_interruptible - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible(wq_head, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_INTERRUPTIBLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a signal.
 */
#define wait_event_interruptible_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_interruptible_timeout(wq_head,                \
                                                condition, timeout);                \
        __ret;                                                                        \
})

#define __wait_event_hrtimeout(wq_head, condition, timeout, state)                \
({                                                                                \
        int __ret = 0;                                                                \
        struct hrtimer_sleeper __t;                                                \
                                                                                \
        hrtimer_setup_sleeper_on_stack(&__t, CLOCK_MONOTONIC,                        \
                                       HRTIMER_MODE_REL);                        \
        if ((timeout) != KTIME_MAX) {                                                \
                hrtimer_set_expires_range_ns(&__t.timer, timeout,                \
                                        current->timer_slack_ns);                \
                hrtimer_sleeper_start_expires(&__t, HRTIMER_MODE_REL);                \
        }                                                                        \
                                                                                \
        __ret = ___wait_event(wq_head, condition, state, 0, 0,                        \
                if (!__t.task) {                                                \
                        __ret = -ETIME;                                                \
                        break;                                                        \
                }                                                                \
                schedule());                                                        \
                                                                                \
        hrtimer_cancel(&__t.timer);                                                \
        destroy_hrtimer_on_stack(&__t.timer);                                        \
        __ret;                                                                        \
})

/**
 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, or -ETIME if the timeout
 * elapsed.
 */
#define wait_event_hrtimeout(wq_head, condition, timeout)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq_head, condition, timeout,        \
                                               TASK_UNINTERRUPTIBLE);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, as a ktime_t
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 * interrupted by a signal, or -ETIME if the timeout elapsed.
 */
#define wait_event_interruptible_hrtimeout(wq, condition, timeout)                \
({                                                                                \
        long __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_hrtimeout(wq, condition, timeout,                \
                                               TASK_INTERRUPTIBLE);                \
        __ret;                                                                        \
})

#define __wait_event_interruptible_exclusive(wq, condition)                        \
        ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                        \
                      schedule())

#define wait_event_interruptible_exclusive(wq, condition)                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_exclusive(wq, condition);        \
        __ret;                                                                        \
})

#define __wait_event_killable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                        \
                      schedule())

#define wait_event_killable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable_exclusive(wq, condition);                \
        __ret;                                                                        \
})


#define __wait_event_freezable_exclusive(wq, condition)                                \
        ___wait_event(wq, condition, (TASK_INTERRUPTIBLE|TASK_FREEZABLE), 1, 0,\
                        schedule())

#define wait_event_freezable_exclusive(wq, condition)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_freezable_exclusive(wq, condition);        \
        __ret;                                                                        \
})

/**
 * wait_event_idle - wait for a condition without contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle(wq_head, condition)                                        \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule());        \
} while (0)

/**
 * wait_event_idle_exclusive - wait for a condition with contributing to system load
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 */
#define wait_event_idle_exclusive(wq_head, condition)                                \
do {                                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule());        \
} while (0)

#define __wait_event_idle_timeout(wq_head, condition, timeout)                        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 0, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_timeout(wq_head, condition, timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_timeout(wq_head, condition, timeout);        \
        __ret;                                                                        \
})

#define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_IDLE, 1, timeout,                                        \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_IDLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus if other processes wait on the same list, when this
 * process is woken further processes are not considered.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * or the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed.
 */
#define wait_event_idle_exclusive_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
        __ret;                                                                        \
})

extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);

#define __wait_event_interruptible_locked(wq, condition, exclusive, fn)                \
({                                                                                \
        int __ret;                                                                \
        DEFINE_WAIT(__wait);                                                        \
        if (exclusive)                                                                \
                __wait.flags |= WQ_FLAG_EXCLUSIVE;                                \
        do {                                                                        \
                __ret = fn(&(wq), &__wait);                                        \
                if (__ret)                                                        \
                        break;                                                        \
        } while (!(condition));                                                        \
        __remove_wait_queue(&(wq), &__wait);                                        \
        __set_current_state(TASK_RUNNING);                                        \
        __ret;                                                                        \
})


/**
 * wait_event_interruptible_locked - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked(wq, condition)                                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))

/**
 * wait_event_interruptible_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_locked_irq(wq, condition)                        \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))

/**
 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock()/spin_unlock()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))

/**
 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 * @wq: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq is woken up.
 *
 * It must be called with wq.lock being held.  This spinlock is
 * unlocked while sleeping but @condition testing is done while lock
 * is held and when this macro exits the lock is held.
 *
 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 * functions which must match the way they are locked/unlocked outside
 * of this macro.
 *
 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 * set thus when other process waits process on the list if this
 * process is awaken further processes are not considered.
 *
 * wake_up_locked() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_exclusive_locked_irq(wq, condition)                \
        ((condition)                                                                \
         ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))


#define __wait_event_killable(wq, condition)                                        \
        ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())

/**
 * wait_event_killable - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a
 * signal and 0 if @condition evaluated to true.
 */
#define wait_event_killable(wq_head, condition)                                        \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_killable(wq_head, condition);                \
        __ret;                                                                        \
})

#define __wait_event_state(wq, condition, state)                                \
        ___wait_event(wq, condition, state, 0, 0, schedule())

/**
 * wait_event_state - sleep until a condition gets true
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @state: state to sleep in
 *
 * The process is put to sleep (@state) until the @condition evaluates to true
 * or a signal is received (when allowed by @state).  The @condition is checked
 * each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * The function will return -ERESTARTSYS if it was interrupted by a signal
 * (when allowed by @state) and 0 if @condition evaluated to true.
 */
#define wait_event_state(wq_head, condition, state)                                \
({                                                                                \
        int __ret = 0;                                                                \
        might_sleep();                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_state(wq_head, condition, state);                \
        __ret;                                                                        \
})

#define __wait_event_killable_timeout(wq_head, condition, timeout)                \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      TASK_KILLABLE, 0, timeout,                                \
                      __ret = schedule_timeout(__ret))

/**
 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_KILLABLE) until the
 * @condition evaluates to true or a kill signal is received.
 * The @condition is checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * Returns:
 * 0 if the @condition evaluated to %false after the @timeout elapsed,
 * 1 if the @condition evaluated to %true after the @timeout elapsed,
 * the remaining jiffies (at least 1) if the @condition evaluated
 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 * interrupted by a kill signal.
 *
 * Only kill signals interrupt this process.
 */
#define wait_event_killable_timeout(wq_head, condition, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        might_sleep();                                                                \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_killable_timeout(wq_head,                        \
                                                condition, timeout);                \
        __ret;                                                                        \
})


#define __wait_event_lock_irq(wq_head, condition, lock, cmd)                        \
        (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,        \
                            spin_unlock_irq(&lock);                                \
                            cmd;                                                \
                            schedule();                                                \
                            spin_lock_irq(&lock))

/**
 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 *                             condition is checked under the lock. This
 *                             is expected to be called with the lock
 *                             taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd
 *          and schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 */
#define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                        \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, cmd);                        \
} while (0)

/**
 * wait_event_lock_irq - sleep until a condition gets true. The
 *                         condition is checked under the lock. This
 *                         is expected to be called with the lock
 *                         taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 * @condition evaluates to true. The @condition is checked each time
 * the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 */
#define wait_event_lock_irq(wq_head, condition, lock)                                \
do {                                                                                \
        if (condition)                                                                \
                break;                                                                \
        __wait_event_lock_irq(wq_head, condition, lock, );                        \
} while (0)


#define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)        \
        ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,                \
                      spin_unlock_irq(&lock);                                        \
                      cmd;                                                        \
                      schedule();                                                \
                      spin_lock_irq(&lock))

/**
 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected to
 *                be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before cmd and
 *          schedule() and reacquired afterwards.
 * @cmd: a command which is invoked outside the critical section before
 *         sleep
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or a signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before invoking the cmd and going to sleep and is reacquired
 * afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)        \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock, cmd);                \
        __ret;                                                                        \
})

/**
 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 *                The condition is checked under the lock. This is expected
 *                to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The macro will return -ERESTARTSYS if it was interrupted by a signal
 * and 0 if @condition evaluated to true.
 */
#define wait_event_interruptible_lock_irq(wq_head, condition, lock)                \
({                                                                                \
        int __ret = 0;                                                                \
        if (!(condition))                                                        \
                __ret = __wait_event_interruptible_lock_irq(wq_head,                \
                                                condition, lock,);                \
        __ret;                                                                        \
})

#define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state)        \
        ___wait_event(wq_head, ___wait_cond_timeout(condition),                        \
                      state, 0, timeout,                                        \
                      spin_unlock_irq(&lock);                                        \
                      __ret = schedule_timeout(__ret);                                \
                      spin_lock_irq(&lock));

/**
 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 *                true or a timeout elapses. The condition is checked under
 *                the lock. This is expected to be called with the lock taken.
 * @wq_head: the waitqueue to wait on
 * @condition: a C expression for the event to wait for
 * @lock: a locked spinlock_t, which will be released before schedule()
 *          and reacquired afterwards.
 * @timeout: timeout, in jiffies
 *
 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 * @condition evaluates to true or signal is received. The @condition is
 * checked each time the waitqueue @wq_head is woken up.
 *
 * wake_up() has to be called after changing any variable that could
 * change the result of the wait condition.
 *
 * This is supposed to be called while holding the lock. The lock is
 * dropped before going to sleep and is reacquired afterwards.
 *
 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 * was interrupted by a signal, and the remaining jiffies otherwise
 * if the condition evaluated to true before the timeout elapsed.
 */
#define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,        \
                                                  timeout)                        \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_INTERRUPTIBLE);                        \
        __ret;                                                                        \
})

#define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout)                \
({                                                                                \
        long __ret = timeout;                                                        \
        if (!___wait_cond_timeout(condition))                                        \
                __ret = __wait_event_lock_irq_timeout(                                \
                                        wq_head, condition, lock, timeout,        \
                                        TASK_UNINTERRUPTIBLE);                        \
        __ret;                                                                        \
})

/*
 * Waitqueues which are removed from the waitqueue_head at wakeup time
 */
void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);

#define DEFINE_WAIT_FUNC(name, function)                                        \
        struct wait_queue_entry name = {                                        \
                .private        = current,                                        \
                .func                = function,                                        \
                .entry                = LIST_HEAD_INIT((name).entry),                        \
        }

#define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)

#define init_wait_func(wait, function)                                                \
        do {                                                                        \
                (wait)->private = current;                                        \
                (wait)->func = function;                                        \
                INIT_LIST_HEAD(&(wait)->entry);                                        \
                (wait)->flags = 0;                                                \
        } while (0)

#define init_wait(wait)        init_wait_func(wait, autoremove_wake_function)

typedef int (*task_call_f)(struct task_struct *p, void *arg);
extern int task_call_func(struct task_struct *p, task_call_f func, void *arg);

#endif /* _LINUX_WAIT_H */

























































































































































































































































































































































































































































   49 


























































































































   99 







































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SWAPOPS_H
#define _LINUX_SWAPOPS_H

#include <linux/radix-tree.h>
#include <linux/bug.h>
#include <linux/mm_types.h>

#ifdef CONFIG_MMU

#ifdef CONFIG_SWAP
#include <linux/swapfile.h>
#endif        /* CONFIG_SWAP */

/*
 * swapcache pages are stored in the swapper_space radix tree.  We want to
 * get good packing density in that tree, so the index should be dense in
 * the low-order bits.
 *
 * We arrange the `type' and `offset' fields so that `type' is at the six
 * high-order bits of the swp_entry_t and `offset' is right-aligned in the
 * remaining bits.  Although `type' itself needs only five bits, we allow for
 * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry().
 *
 * swp_entry_t's are *never* stored anywhere in their arch-dependent format.
 */
#define SWP_TYPE_SHIFT        (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
#define SWP_OFFSET_MASK        ((1UL << SWP_TYPE_SHIFT) - 1)

/*
 * Definitions only for PFN swap entries (see is_pfn_swap_entry()).  To
 * store PFN, we only need SWP_PFN_BITS bits.  Each of the pfn swap entries
 * can use the extra bits to store other information besides PFN.
 */
#ifdef MAX_PHYSMEM_BITS
#define SWP_PFN_BITS                (MAX_PHYSMEM_BITS - PAGE_SHIFT)
#else  /* MAX_PHYSMEM_BITS */
#define SWP_PFN_BITS                min_t(int, \
                                      sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \
                                      SWP_TYPE_SHIFT)
#endif        /* MAX_PHYSMEM_BITS */
#define SWP_PFN_MASK                (BIT(SWP_PFN_BITS) - 1)

/**
 * Migration swap entry specific bitfield definitions.  Layout:
 *
 *   |----------+--------------------|
 *   | swp_type | swp_offset         |
 *   |----------+--------+-+-+-------|
 *   |          | resv   |D|A|  PFN  |
 *   |----------+--------+-+-+-------|
 *
 * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A)
 * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D)
 *
 * Note: A/D bits will be stored in migration entries iff there're enough
 * free bits in arch specific swp offset.  By default we'll ignore A/D bits
 * when migrating a page.  Please refer to migration_entry_supports_ad()
 * for more information.  If there're more bits besides PFN and A/D bits,
 * they should be reserved and always be zeros.
 */
#define SWP_MIG_YOUNG_BIT                (SWP_PFN_BITS)
#define SWP_MIG_DIRTY_BIT                (SWP_PFN_BITS + 1)
#define SWP_MIG_TOTAL_BITS                (SWP_PFN_BITS + 2)

#define SWP_MIG_YOUNG                        BIT(SWP_MIG_YOUNG_BIT)
#define SWP_MIG_DIRTY                        BIT(SWP_MIG_DIRTY_BIT)

static inline bool is_pfn_swap_entry(swp_entry_t entry);

/* Clear all flags but only keep swp_entry_t related information */
static inline pte_t pte_swp_clear_flags(pte_t pte)
{
        if (pte_swp_exclusive(pte))
                pte = pte_swp_clear_exclusive(pte);
        if (pte_swp_soft_dirty(pte))
                pte = pte_swp_clear_soft_dirty(pte);
        if (pte_swp_uffd_wp(pte))
                pte = pte_swp_clear_uffd_wp(pte);
        return pte;
}

/*
 * Store a type+offset into a swp_entry_t in an arch-independent format
 */
static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset)
{
        swp_entry_t ret;

        ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK);
        return ret;
}

/*
 * Extract the `type' field from a swp_entry_t.  The swp_entry_t is in
 * arch-independent format
 */
static inline unsigned swp_type(swp_entry_t entry)
{
        return (entry.val >> SWP_TYPE_SHIFT);
}

/*
 * Extract the `offset' field from a swp_entry_t.  The swp_entry_t is in
 * arch-independent format
 */
static inline pgoff_t swp_offset(swp_entry_t entry)
{
        return entry.val & SWP_OFFSET_MASK;
}

/*
 * This should only be called upon a pfn swap entry to get the PFN stored
 * in the swap entry.  Please refers to is_pfn_swap_entry() for definition
 * of pfn swap entry.
 */
static inline unsigned long swp_offset_pfn(swp_entry_t entry)
{
        VM_BUG_ON(!is_pfn_swap_entry(entry));
        return swp_offset(entry) & SWP_PFN_MASK;
}

/* check whether a pte points to a swap entry */
static inline int is_swap_pte(pte_t pte)
{
        return !pte_none(pte) && !pte_present(pte);
}

/*
 * Convert the arch-dependent pte representation of a swp_entry_t into an
 * arch-independent swp_entry_t.
 */
static inline swp_entry_t pte_to_swp_entry(pte_t pte)
{
        swp_entry_t arch_entry;

        pte = pte_swp_clear_flags(pte);
        arch_entry = __pte_to_swp_entry(pte);
        return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
}

/*
 * Convert the arch-independent representation of a swp_entry_t into the
 * arch-dependent pte representation.
 */
static inline pte_t swp_entry_to_pte(swp_entry_t entry)
{
        swp_entry_t arch_entry;

        arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
        return __swp_entry_to_pte(arch_entry);
}

static inline swp_entry_t radix_to_swp_entry(void *arg)
{
        swp_entry_t entry;

        entry.val = xa_to_value(arg);
        return entry;
}

static inline void *swp_to_radix_entry(swp_entry_t entry)
{
        return xa_mk_value(entry.val);
}

#if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
{
        return swp_entry(SWP_DEVICE_READ, offset);
}

static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
{
        return swp_entry(SWP_DEVICE_WRITE, offset);
}

static inline bool is_device_private_entry(swp_entry_t entry)
{
        int type = swp_type(entry);
        return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE;
}

static inline bool is_writable_device_private_entry(swp_entry_t entry)
{
        return unlikely(swp_type(entry) == SWP_DEVICE_WRITE);
}

static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset)
{
        return swp_entry(SWP_DEVICE_EXCLUSIVE, offset);
}

static inline bool is_device_exclusive_entry(swp_entry_t entry)
{
        return swp_type(entry) == SWP_DEVICE_EXCLUSIVE;
}

#else /* CONFIG_DEVICE_PRIVATE */
static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline bool is_device_private_entry(swp_entry_t entry)
{
        return false;
}

static inline bool is_writable_device_private_entry(swp_entry_t entry)
{
        return false;
}

static inline swp_entry_t make_device_exclusive_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline bool is_device_exclusive_entry(swp_entry_t entry)
{
        return false;
}

#endif /* CONFIG_DEVICE_PRIVATE */

#ifdef CONFIG_MIGRATION
static inline int is_migration_entry(swp_entry_t entry)
{
        return unlikely(swp_type(entry) == SWP_MIGRATION_READ ||
                        swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE ||
                        swp_type(entry) == SWP_MIGRATION_WRITE);
}

static inline int is_writable_migration_entry(swp_entry_t entry)
{
        return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE);
}

static inline int is_readable_migration_entry(swp_entry_t entry)
{
        return unlikely(swp_type(entry) == SWP_MIGRATION_READ);
}

static inline int is_readable_exclusive_migration_entry(swp_entry_t entry)
{
        return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE);
}

static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
{
        return swp_entry(SWP_MIGRATION_READ, offset);
}

static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
{
        return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset);
}

static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
{
        return swp_entry(SWP_MIGRATION_WRITE, offset);
}

/*
 * Returns whether the host has large enough swap offset field to support
 * carrying over pgtable A/D bits for page migrations.  The result is
 * pretty much arch specific.
 */
static inline bool migration_entry_supports_ad(void)
{
#ifdef CONFIG_SWAP
        return swap_migration_ad_supported;
#else  /* CONFIG_SWAP */
        return false;
#endif        /* CONFIG_SWAP */
}

static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
{
        if (migration_entry_supports_ad())
                return swp_entry(swp_type(entry),
                                 swp_offset(entry) | SWP_MIG_YOUNG);
        return entry;
}

static inline bool is_migration_entry_young(swp_entry_t entry)
{
        if (migration_entry_supports_ad())
                return swp_offset(entry) & SWP_MIG_YOUNG;
        /* Keep the old behavior of aging page after migration */
        return false;
}

static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
{
        if (migration_entry_supports_ad())
                return swp_entry(swp_type(entry),
                                 swp_offset(entry) | SWP_MIG_DIRTY);
        return entry;
}

static inline bool is_migration_entry_dirty(swp_entry_t entry)
{
        if (migration_entry_supports_ad())
                return swp_offset(entry) & SWP_MIG_DIRTY;
        /* Keep the old behavior of clean page after migration */
        return false;
}

extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
                                        unsigned long address);
extern void migration_entry_wait_huge(struct vm_area_struct *vma, unsigned long addr, pte_t *pte);
#else  /* CONFIG_MIGRATION */
static inline swp_entry_t make_readable_migration_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline swp_entry_t make_writable_migration_entry(pgoff_t offset)
{
        return swp_entry(0, 0);
}

static inline int is_migration_entry(swp_entry_t swp)
{
        return 0;
}

static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
                                        unsigned long address) { }
static inline void migration_entry_wait_huge(struct vm_area_struct *vma,
                                             unsigned long addr, pte_t *pte) { }
static inline int is_writable_migration_entry(swp_entry_t entry)
{
        return 0;
}
static inline int is_readable_migration_entry(swp_entry_t entry)
{
        return 0;
}

static inline swp_entry_t make_migration_entry_young(swp_entry_t entry)
{
        return entry;
}

static inline bool is_migration_entry_young(swp_entry_t entry)
{
        return false;
}

static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry)
{
        return entry;
}

static inline bool is_migration_entry_dirty(swp_entry_t entry)
{
        return false;
}
#endif        /* CONFIG_MIGRATION */

#ifdef CONFIG_MEMORY_FAILURE

/*
 * Support for hardware poisoned pages
 */
static inline swp_entry_t make_hwpoison_entry(struct page *page)
{
        BUG_ON(!PageLocked(page));
        return swp_entry(SWP_HWPOISON, page_to_pfn(page));
}

static inline int is_hwpoison_entry(swp_entry_t entry)
{
        return swp_type(entry) == SWP_HWPOISON;
}

#else

static inline swp_entry_t make_hwpoison_entry(struct page *page)
{
        return swp_entry(0, 0);
}

static inline int is_hwpoison_entry(swp_entry_t swp)
{
        return 0;
}
#endif

typedef unsigned long pte_marker;

#define  PTE_MARKER_UFFD_WP                        BIT(0)
/*
 * "Poisoned" here is meant in the very general sense of "future accesses are
 * invalid", instead of referring very specifically to hardware memory errors.
 * This marker is meant to represent any of various different causes of this.
 *
 * Note that, when encountered by the faulting logic, PTEs with this marker will
 * result in VM_FAULT_HWPOISON and thus regardless trigger hardware memory error
 * logic.
 */
#define  PTE_MARKER_POISONED                        BIT(1)
/*
 * Indicates that, on fault, this PTE will case a SIGSEGV signal to be
 * sent. This means guard markers behave in effect as if the region were mapped
 * PROT_NONE, rather than if they were a memory hole or equivalent.
 */
#define  PTE_MARKER_GUARD                        BIT(2)
#define  PTE_MARKER_MASK                        (BIT(3) - 1)

static inline swp_entry_t make_pte_marker_entry(pte_marker marker)
{
        return swp_entry(SWP_PTE_MARKER, marker);
}

static inline bool is_pte_marker_entry(swp_entry_t entry)
{
        return swp_type(entry) == SWP_PTE_MARKER;
}

static inline pte_marker pte_marker_get(swp_entry_t entry)
{
        return swp_offset(entry) & PTE_MARKER_MASK;
}

static inline bool is_pte_marker(pte_t pte)
{
        return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte));
}

static inline pte_t make_pte_marker(pte_marker marker)
{
        return swp_entry_to_pte(make_pte_marker_entry(marker));
}

static inline swp_entry_t make_poisoned_swp_entry(void)
{
        return make_pte_marker_entry(PTE_MARKER_POISONED);
}

static inline int is_poisoned_swp_entry(swp_entry_t entry)
{
        return is_pte_marker_entry(entry) &&
            (pte_marker_get(entry) & PTE_MARKER_POISONED);

}

static inline swp_entry_t make_guard_swp_entry(void)
{
        return make_pte_marker_entry(PTE_MARKER_GUARD);
}

static inline int is_guard_swp_entry(swp_entry_t entry)
{
        return is_pte_marker_entry(entry) &&
                (pte_marker_get(entry) & PTE_MARKER_GUARD);
}

/*
 * This is a special version to check pte_none() just to cover the case when
 * the pte is a pte marker.  It existed because in many cases the pte marker
 * should be seen as a none pte; it's just that we have stored some information
 * onto the none pte so it becomes not-none any more.
 *
 * It should be used when the pte is file-backed, ram-based and backing
 * userspace pages, like shmem.  It is not needed upon pgtables that do not
 * support pte markers at all.  For example, it's not needed on anonymous
 * memory, kernel-only memory (including when the system is during-boot),
 * non-ram based generic file-system.  It's fine to be used even there, but the
 * extra pte marker check will be pure overhead.
 */
static inline int pte_none_mostly(pte_t pte)
{
        return pte_none(pte) || is_pte_marker(pte);
}

static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
{
        struct page *p = pfn_to_page(swp_offset_pfn(entry));

        /*
         * Any use of migration entries may only occur while the
         * corresponding page is locked
         */
        BUG_ON(is_migration_entry(entry) && !PageLocked(p));

        return p;
}

static inline struct folio *pfn_swap_entry_folio(swp_entry_t entry)
{
        struct folio *folio = pfn_folio(swp_offset_pfn(entry));

        /*
         * Any use of migration entries may only occur while the
         * corresponding folio is locked
         */
        BUG_ON(is_migration_entry(entry) && !folio_test_locked(folio));

        return folio;
}

/*
 * A pfn swap entry is a special type of swap entry that always has a pfn stored
 * in the swap offset. They can either be used to represent unaddressable device
 * memory, to restrict access to a page undergoing migration or to represent a
 * pfn which has been hwpoisoned and unmapped.
 */
static inline bool is_pfn_swap_entry(swp_entry_t entry)
{
        /* Make sure the swp offset can always store the needed fields */
        BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);

        return is_migration_entry(entry) || is_device_private_entry(entry) ||
               is_device_exclusive_entry(entry) || is_hwpoison_entry(entry);
}

struct page_vma_mapped_walk;

#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
                struct page *page);

extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
                struct page *new);

extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd);

static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
{
        swp_entry_t arch_entry;

        if (pmd_swp_soft_dirty(pmd))
                pmd = pmd_swp_clear_soft_dirty(pmd);
        if (pmd_swp_uffd_wp(pmd))
                pmd = pmd_swp_clear_uffd_wp(pmd);
        arch_entry = __pmd_to_swp_entry(pmd);
        return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry));
}

static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
{
        swp_entry_t arch_entry;

        arch_entry = __swp_entry(swp_type(entry), swp_offset(entry));
        return __swp_entry_to_pmd(arch_entry);
}

static inline int is_pmd_migration_entry(pmd_t pmd)
{
        return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd));
}
#else  /* CONFIG_ARCH_ENABLE_THP_MIGRATION */
static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
                struct page *page)
{
        BUILD_BUG();
}

static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw,
                struct page *new)
{
        BUILD_BUG();
}

static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { }

static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd)
{
        return swp_entry(0, 0);
}

static inline pmd_t swp_entry_to_pmd(swp_entry_t entry)
{
        return __pmd(0);
}

static inline int is_pmd_migration_entry(pmd_t pmd)
{
        return 0;
}
#endif  /* CONFIG_ARCH_ENABLE_THP_MIGRATION */

static inline int non_swap_entry(swp_entry_t entry)
{
        return swp_type(entry) >= MAX_SWAPFILES;
}

#endif /* CONFIG_MMU */
#endif /* _LINUX_SWAPOPS_H */



































































    2 








    2 











































































































































































































































































    2 







    2 












    2 







    2 




    2 

    2 










































    2 


    2 
    2 


    2 




    2 























































































































































































































































































































































































    2 

    2 










    2 





































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        Generic address resolution entity
 *
 *        Authors:
 *        Pedro Roque                <roque@di.fc.ul.pt>
 *        Alexey Kuznetsov        <kuznet@ms2.inr.ac.ru>
 *
 *        Fixes:
 *        Vitaly E. Lavrov        releasing NULL neighbor in neigh_add.
 *        Harald Welte                Add neighbour cache statistics like rtstat
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/socket.h>
#include <linux/netdevice.h>
#include <linux/proc_fs.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif
#include <linux/times.h>
#include <net/net_namespace.h>
#include <net/neighbour.h>
#include <net/arp.h>
#include <net/dst.h>
#include <net/sock.h>
#include <net/netevent.h>
#include <net/netlink.h>
#include <linux/rtnetlink.h>
#include <linux/random.h>
#include <linux/string.h>
#include <linux/log2.h>
#include <linux/inetdevice.h>
#include <net/addrconf.h>

#include <trace/events/neigh.h>

#define NEIGH_DEBUG 1
#define neigh_dbg(level, fmt, ...)                \
do {                                                \
        if (level <= NEIGH_DEBUG)                \
                pr_debug(fmt, ##__VA_ARGS__);        \
} while (0)

#define PNEIGH_HASHMASK                0xF

static void neigh_timer_handler(struct timer_list *t);
static void __neigh_notify(struct neighbour *n, int type, int flags,
                           u32 pid);
static void neigh_update_notify(struct neighbour *neigh, u32 nlmsg_pid);
static int pneigh_ifdown_and_unlock(struct neigh_table *tbl,
                                    struct net_device *dev);

#ifdef CONFIG_PROC_FS
static const struct seq_operations neigh_stat_seq_ops;
#endif

static struct hlist_head *neigh_get_dev_table(struct net_device *dev, int family)
{
        int i;

        switch (family) {
        default:
                DEBUG_NET_WARN_ON_ONCE(1);
                fallthrough; /* to avoid panic by null-ptr-deref */
        case AF_INET:
                i = NEIGH_ARP_TABLE;
                break;
        case AF_INET6:
                i = NEIGH_ND_TABLE;
                break;
        }

        return &dev->neighbours[i];
}

/*
   Neighbour hash table buckets are protected with rwlock tbl->lock.

   - All the scans/updates to hash buckets MUST be made under this lock.
   - NOTHING clever should be made under this lock: no callbacks
     to protocol backends, no attempts to send something to network.
     It will result in deadlocks, if backend/driver wants to use neighbour
     cache.
   - If the entry requires some non-trivial actions, increase
     its reference count and release table lock.

   Neighbour entries are protected:
   - with reference count.
   - with rwlock neigh->lock

   Reference count prevents destruction.

   neigh->lock mainly serializes ll address data and its validity state.
   However, the same lock is used to protect another entry fields:
    - timer
    - resolution queue

   Again, nothing clever shall be made under neigh->lock,
   the most complicated procedure, which we allow is dev->hard_header.
   It is supposed, that dev->hard_header is simplistic and does
   not make callbacks to neighbour tables.
 */

static int neigh_blackhole(struct neighbour *neigh, struct sk_buff *skb)
{
        kfree_skb(skb);
        return -ENETDOWN;
}

static void neigh_cleanup_and_release(struct neighbour *neigh)
{
        trace_neigh_cleanup_and_release(neigh, 0);
        __neigh_notify(neigh, RTM_DELNEIGH, 0, 0);
        call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
        neigh_release(neigh);
}

/*
 * It is random distribution in the interval (1/2)*base...(3/2)*base.
 * It corresponds to default IPv6 settings and is not overridable,
 * because it is really reasonable choice.
 */

unsigned long neigh_rand_reach_time(unsigned long base)
{
        return base ? get_random_u32_below(base) + (base >> 1) : 0;
}
EXPORT_SYMBOL(neigh_rand_reach_time);

static void neigh_mark_dead(struct neighbour *n)
{
        n->dead = 1;
        if (!list_empty(&n->gc_list)) {
                list_del_init(&n->gc_list);
                atomic_dec(&n->tbl->gc_entries);
        }
        if (!list_empty(&n->managed_list))
                list_del_init(&n->managed_list);
}

static void neigh_update_gc_list(struct neighbour *n)
{
        bool on_gc_list, exempt_from_gc;

        write_lock_bh(&n->tbl->lock);
        write_lock(&n->lock);
        if (n->dead)
                goto out;

        /* remove from the gc list if new state is permanent or if neighbor
         * is externally learned; otherwise entry should be on the gc list
         */
        exempt_from_gc = n->nud_state & NUD_PERMANENT ||
                         n->flags & NTF_EXT_LEARNED;
        on_gc_list = !list_empty(&n->gc_list);

        if (exempt_from_gc && on_gc_list) {
                list_del_init(&n->gc_list);
                atomic_dec(&n->tbl->gc_entries);
        } else if (!exempt_from_gc && !on_gc_list) {
                /* add entries to the tail; cleaning removes from the front */
                list_add_tail(&n->gc_list, &n->tbl->gc_list);
                atomic_inc(&n->tbl->gc_entries);
        }
out:
        write_unlock(&n->lock);
        write_unlock_bh(&n->tbl->lock);
}

static void neigh_update_managed_list(struct neighbour *n)
{
        bool on_managed_list, add_to_managed;

        write_lock_bh(&n->tbl->lock);
        write_lock(&n->lock);
        if (n->dead)
                goto out;

        add_to_managed = n->flags & NTF_MANAGED;
        on_managed_list = !list_empty(&n->managed_list);

        if (!add_to_managed && on_managed_list)
                list_del_init(&n->managed_list);
        else if (add_to_managed && !on_managed_list)
                list_add_tail(&n->managed_list, &n->tbl->managed_list);
out:
        write_unlock(&n->lock);
        write_unlock_bh(&n->tbl->lock);
}

static void neigh_update_flags(struct neighbour *neigh, u32 flags, int *notify,
                               bool *gc_update, bool *managed_update)
{
        u32 ndm_flags, old_flags = neigh->flags;

        if (!(flags & NEIGH_UPDATE_F_ADMIN))
                return;

        ndm_flags  = (flags & NEIGH_UPDATE_F_EXT_LEARNED) ? NTF_EXT_LEARNED : 0;
        ndm_flags |= (flags & NEIGH_UPDATE_F_MANAGED) ? NTF_MANAGED : 0;

        if ((old_flags ^ ndm_flags) & NTF_EXT_LEARNED) {
                if (ndm_flags & NTF_EXT_LEARNED)
                        neigh->flags |= NTF_EXT_LEARNED;
                else
                        neigh->flags &= ~NTF_EXT_LEARNED;
                *notify = 1;
                *gc_update = true;
        }
        if ((old_flags ^ ndm_flags) & NTF_MANAGED) {
                if (ndm_flags & NTF_MANAGED)
                        neigh->flags |= NTF_MANAGED;
                else
                        neigh->flags &= ~NTF_MANAGED;
                *notify = 1;
                *managed_update = true;
        }
}

bool neigh_remove_one(struct neighbour *n)
{
        bool retval = false;

        write_lock(&n->lock);
        if (refcount_read(&n->refcnt) == 1) {
                hlist_del_rcu(&n->hash);
                hlist_del_rcu(&n->dev_list);
                neigh_mark_dead(n);
                retval = true;
        }
        write_unlock(&n->lock);
        if (retval)
                neigh_cleanup_and_release(n);
        return retval;
}

static int neigh_forced_gc(struct neigh_table *tbl)
{
        int max_clean = atomic_read(&tbl->gc_entries) -
                        READ_ONCE(tbl->gc_thresh2);
        u64 tmax = ktime_get_ns() + NSEC_PER_MSEC;
        unsigned long tref = jiffies - 5 * HZ;
        struct neighbour *n, *tmp;
        int shrunk = 0;
        int loop = 0;

        NEIGH_CACHE_STAT_INC(tbl, forced_gc_runs);

        write_lock_bh(&tbl->lock);

        list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) {
                if (refcount_read(&n->refcnt) == 1) {
                        bool remove = false;

                        write_lock(&n->lock);
                        if ((n->nud_state == NUD_FAILED) ||
                            (n->nud_state == NUD_NOARP) ||
                            (tbl->is_multicast &&
                             tbl->is_multicast(n->primary_key)) ||
                            !time_in_range(n->updated, tref, jiffies))
                                remove = true;
                        write_unlock(&n->lock);

                        if (remove && neigh_remove_one(n))
                                shrunk++;
                        if (shrunk >= max_clean)
                                break;
                        if (++loop == 16) {
                                if (ktime_get_ns() > tmax)
                                        goto unlock;
                                loop = 0;
                        }
                }
        }

        WRITE_ONCE(tbl->last_flush, jiffies);
unlock:
        write_unlock_bh(&tbl->lock);

        return shrunk;
}

static void neigh_add_timer(struct neighbour *n, unsigned long when)
{
        /* Use safe distance from the jiffies - LONG_MAX point while timer
         * is running in DELAY/PROBE state but still show to user space
         * large times in the past.
         */
        unsigned long mint = jiffies - (LONG_MAX - 86400 * HZ);

        neigh_hold(n);
        if (!time_in_range(n->confirmed, mint, jiffies))
                n->confirmed = mint;
        if (time_before(n->used, n->confirmed))
                n->used = n->confirmed;
        if (unlikely(mod_timer(&n->timer, when))) {
                printk("NEIGH: BUG, double timer add, state is %x\n",
                       n->nud_state);
                dump_stack();
        }
}

static int neigh_del_timer(struct neighbour *n)
{
        if ((n->nud_state & NUD_IN_TIMER) &&
            timer_delete(&n->timer)) {
                neigh_release(n);
                return 1;
        }
        return 0;
}

static struct neigh_parms *neigh_get_dev_parms_rcu(struct net_device *dev,
                                                   int family)
{
        switch (family) {
        case AF_INET:
                return __in_dev_arp_parms_get_rcu(dev);
        case AF_INET6:
                return __in6_dev_nd_parms_get_rcu(dev);
        }
        return NULL;
}

static void neigh_parms_qlen_dec(struct net_device *dev, int family)
{
        struct neigh_parms *p;

        rcu_read_lock();
        p = neigh_get_dev_parms_rcu(dev, family);
        if (p)
                p->qlen--;
        rcu_read_unlock();
}

static void pneigh_queue_purge(struct sk_buff_head *list, struct net *net,
                               int family)
{
        struct sk_buff_head tmp;
        unsigned long flags;
        struct sk_buff *skb;

        skb_queue_head_init(&tmp);
        spin_lock_irqsave(&list->lock, flags);
        skb = skb_peek(list);
        while (skb != NULL) {
                struct sk_buff *skb_next = skb_peek_next(skb, list);
                struct net_device *dev = skb->dev;

                if (net == NULL || net_eq(dev_net(dev), net)) {
                        neigh_parms_qlen_dec(dev, family);
                        __skb_unlink(skb, list);
                        __skb_queue_tail(&tmp, skb);
                }
                skb = skb_next;
        }
        spin_unlock_irqrestore(&list->lock, flags);

        while ((skb = __skb_dequeue(&tmp))) {
                dev_put(skb->dev);
                kfree_skb(skb);
        }
}

static void neigh_flush_dev(struct neigh_table *tbl, struct net_device *dev,
                            bool skip_perm)
{
        struct hlist_head *dev_head;
        struct hlist_node *tmp;
        struct neighbour *n;

        dev_head = neigh_get_dev_table(dev, tbl->family);

        hlist_for_each_entry_safe(n, tmp, dev_head, dev_list) {
                if (skip_perm && n->nud_state & NUD_PERMANENT)
                        continue;

                hlist_del_rcu(&n->hash);
                hlist_del_rcu(&n->dev_list);
                write_lock(&n->lock);
                neigh_del_timer(n);
                neigh_mark_dead(n);
                if (refcount_read(&n->refcnt) != 1) {
                        /* The most unpleasant situation.
                         * We must destroy neighbour entry,
                         * but someone still uses it.
                         *
                         * The destroy will be delayed until
                         * the last user releases us, but
                         * we must kill timers etc. and move
                         * it to safe state.
                         */
                        __skb_queue_purge(&n->arp_queue);
                        n->arp_queue_len_bytes = 0;
                        WRITE_ONCE(n->output, neigh_blackhole);
                        if (n->nud_state & NUD_VALID)
                                n->nud_state = NUD_NOARP;
                        else
                                n->nud_state = NUD_NONE;
                        neigh_dbg(2, "neigh %p is stray\n", n);
                }
                write_unlock(&n->lock);
                neigh_cleanup_and_release(n);
        }
}

void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev)
{
        write_lock_bh(&tbl->lock);
        neigh_flush_dev(tbl, dev, false);
        write_unlock_bh(&tbl->lock);
}
EXPORT_SYMBOL(neigh_changeaddr);

static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev,
                          bool skip_perm)
{
        write_lock_bh(&tbl->lock);
        neigh_flush_dev(tbl, dev, skip_perm);
        pneigh_ifdown_and_unlock(tbl, dev);
        pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL,
                           tbl->family);
        if (skb_queue_empty_lockless(&tbl->proxy_queue))
                timer_delete_sync(&tbl->proxy_timer);
        return 0;
}

int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev)
{
        __neigh_ifdown(tbl, dev, true);
        return 0;
}
EXPORT_SYMBOL(neigh_carrier_down);

int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev)
{
        __neigh_ifdown(tbl, dev, false);
        return 0;
}
EXPORT_SYMBOL(neigh_ifdown);

static struct neighbour *neigh_alloc(struct neigh_table *tbl,
                                     struct net_device *dev,
                                     u32 flags, bool exempt_from_gc)
{
        struct neighbour *n = NULL;
        unsigned long now = jiffies;
        int entries, gc_thresh3;

        if (exempt_from_gc)
                goto do_alloc;

        entries = atomic_inc_return(&tbl->gc_entries) - 1;
        gc_thresh3 = READ_ONCE(tbl->gc_thresh3);
        if (entries >= gc_thresh3 ||
            (entries >= READ_ONCE(tbl->gc_thresh2) &&
             time_after(now, READ_ONCE(tbl->last_flush) + 5 * HZ))) {
                if (!neigh_forced_gc(tbl) && entries >= gc_thresh3) {
                        net_info_ratelimited("%s: neighbor table overflow!\n",
                                             tbl->id);
                        NEIGH_CACHE_STAT_INC(tbl, table_fulls);
                        goto out_entries;
                }
        }

do_alloc:
        n = kzalloc(tbl->entry_size + dev->neigh_priv_len, GFP_ATOMIC);
        if (!n)
                goto out_entries;

        __skb_queue_head_init(&n->arp_queue);
        rwlock_init(&n->lock);
        seqlock_init(&n->ha_lock);
        n->updated          = n->used = now;
        n->nud_state          = NUD_NONE;
        n->output          = neigh_blackhole;
        n->flags          = flags;
        seqlock_init(&n->hh.hh_lock);
        n->parms          = neigh_parms_clone(&tbl->parms);
        timer_setup(&n->timer, neigh_timer_handler, 0);

        NEIGH_CACHE_STAT_INC(tbl, allocs);
        n->tbl                  = tbl;
        refcount_set(&n->refcnt, 1);
        n->dead                  = 1;
        INIT_LIST_HEAD(&n->gc_list);
        INIT_LIST_HEAD(&n->managed_list);

        atomic_inc(&tbl->entries);
out:
        return n;

out_entries:
        if (!exempt_from_gc)
                atomic_dec(&tbl->gc_entries);
        goto out;
}

static void neigh_get_hash_rnd(u32 *x)
{
        *x = get_random_u32() | 1;
}

static struct neigh_hash_table *neigh_hash_alloc(unsigned int shift)
{
        size_t size = (1 << shift) * sizeof(struct hlist_head);
        struct hlist_head *hash_heads;
        struct neigh_hash_table *ret;
        int i;

        ret = kmalloc(sizeof(*ret), GFP_ATOMIC);
        if (!ret)
                return NULL;

        hash_heads = kzalloc(size, GFP_ATOMIC);
        if (!hash_heads) {
                kfree(ret);
                return NULL;
        }
        ret->hash_heads = hash_heads;
        ret->hash_shift = shift;
        for (i = 0; i < NEIGH_NUM_HASH_RND; i++)
                neigh_get_hash_rnd(&ret->hash_rnd[i]);
        return ret;
}

static void neigh_hash_free_rcu(struct rcu_head *head)
{
        struct neigh_hash_table *nht = container_of(head,
                                                    struct neigh_hash_table,
                                                    rcu);

        kfree(nht->hash_heads);
        kfree(nht);
}

static struct neigh_hash_table *neigh_hash_grow(struct neigh_table *tbl,
                                                unsigned long new_shift)
{
        unsigned int i, hash;
        struct neigh_hash_table *new_nht, *old_nht;

        NEIGH_CACHE_STAT_INC(tbl, hash_grows);

        old_nht = rcu_dereference_protected(tbl->nht,
                                            lockdep_is_held(&tbl->lock));
        new_nht = neigh_hash_alloc(new_shift);
        if (!new_nht)
                return old_nht;

        for (i = 0; i < (1 << old_nht->hash_shift); i++) {
                struct hlist_node *tmp;
                struct neighbour *n;

                neigh_for_each_in_bucket_safe(n, tmp, &old_nht->hash_heads[i]) {
                        hash = tbl->hash(n->primary_key, n->dev,
                                         new_nht->hash_rnd);

                        hash >>= (32 - new_nht->hash_shift);

                        hlist_del_rcu(&n->hash);
                        hlist_add_head_rcu(&n->hash, &new_nht->hash_heads[hash]);
                }
        }

        rcu_assign_pointer(tbl->nht, new_nht);
        call_rcu(&old_nht->rcu, neigh_hash_free_rcu);
        return new_nht;
}

struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
                               struct net_device *dev)
{
        struct neighbour *n;

        NEIGH_CACHE_STAT_INC(tbl, lookups);

        rcu_read_lock();
        n = __neigh_lookup_noref(tbl, pkey, dev);
        if (n) {
                if (!refcount_inc_not_zero(&n->refcnt))
                        n = NULL;
                NEIGH_CACHE_STAT_INC(tbl, hits);
        }

        rcu_read_unlock();
        return n;
}
EXPORT_SYMBOL(neigh_lookup);

static struct neighbour *
___neigh_create(struct neigh_table *tbl, const void *pkey,
                struct net_device *dev, u32 flags,
                bool exempt_from_gc, bool want_ref)
{
        u32 hash_val, key_len = tbl->key_len;
        struct neighbour *n1, *rc, *n;
        struct neigh_hash_table *nht;
        int error;

        n = neigh_alloc(tbl, dev, flags, exempt_from_gc);
        trace_neigh_create(tbl, dev, pkey, n, exempt_from_gc);
        if (!n) {
                rc = ERR_PTR(-ENOBUFS);
                goto out;
        }

        memcpy(n->primary_key, pkey, key_len);
        n->dev = dev;
        netdev_hold(dev, &n->dev_tracker, GFP_ATOMIC);

        /* Protocol specific setup. */
        if (tbl->constructor &&        (error = tbl->constructor(n)) < 0) {
                rc = ERR_PTR(error);
                goto out_neigh_release;
        }

        if (dev->netdev_ops->ndo_neigh_construct) {
                error = dev->netdev_ops->ndo_neigh_construct(dev, n);
                if (error < 0) {
                        rc = ERR_PTR(error);
                        goto out_neigh_release;
                }
        }

        /* Device specific setup. */
        if (n->parms->neigh_setup &&
            (error = n->parms->neigh_setup(n)) < 0) {
                rc = ERR_PTR(error);
                goto out_neigh_release;
        }

        n->confirmed = jiffies - (NEIGH_VAR(n->parms, BASE_REACHABLE_TIME) << 1);

        write_lock_bh(&tbl->lock);
        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));

        if (atomic_read(&tbl->entries) > (1 << nht->hash_shift))
                nht = neigh_hash_grow(tbl, nht->hash_shift + 1);

        hash_val = tbl->hash(n->primary_key, dev, nht->hash_rnd) >> (32 - nht->hash_shift);

        if (n->parms->dead) {
                rc = ERR_PTR(-EINVAL);
                goto out_tbl_unlock;
        }

        neigh_for_each_in_bucket(n1, &nht->hash_heads[hash_val]) {
                if (dev == n1->dev && !memcmp(n1->primary_key, n->primary_key, key_len)) {
                        if (want_ref)
                                neigh_hold(n1);
                        rc = n1;
                        goto out_tbl_unlock;
                }
        }

        n->dead = 0;
        if (!exempt_from_gc)
                list_add_tail(&n->gc_list, &n->tbl->gc_list);
        if (n->flags & NTF_MANAGED)
                list_add_tail(&n->managed_list, &n->tbl->managed_list);
        if (want_ref)
                neigh_hold(n);
        hlist_add_head_rcu(&n->hash, &nht->hash_heads[hash_val]);

        hlist_add_head_rcu(&n->dev_list,
                           neigh_get_dev_table(dev, tbl->family));

        write_unlock_bh(&tbl->lock);
        neigh_dbg(2, "neigh %p is created\n", n);
        rc = n;
out:
        return rc;
out_tbl_unlock:
        write_unlock_bh(&tbl->lock);
out_neigh_release:
        if (!exempt_from_gc)
                atomic_dec(&tbl->gc_entries);
        neigh_release(n);
        goto out;
}

struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
                                 struct net_device *dev, bool want_ref)
{
        bool exempt_from_gc = !!(dev->flags & IFF_LOOPBACK);

        return ___neigh_create(tbl, pkey, dev, 0, exempt_from_gc, want_ref);
}
EXPORT_SYMBOL(__neigh_create);

static u32 pneigh_hash(const void *pkey, unsigned int key_len)
{
        u32 hash_val = *(u32 *)(pkey + key_len - 4);
        hash_val ^= (hash_val >> 16);
        hash_val ^= hash_val >> 8;
        hash_val ^= hash_val >> 4;
        hash_val &= PNEIGH_HASHMASK;
        return hash_val;
}

static struct pneigh_entry *__pneigh_lookup_1(struct pneigh_entry *n,
                                              struct net *net,
                                              const void *pkey,
                                              unsigned int key_len,
                                              struct net_device *dev)
{
        while (n) {
                if (!memcmp(n->key, pkey, key_len) &&
                    net_eq(pneigh_net(n), net) &&
                    (n->dev == dev || !n->dev))
                        return n;
                n = n->next;
        }
        return NULL;
}

struct pneigh_entry *__pneigh_lookup(struct neigh_table *tbl,
                struct net *net, const void *pkey, struct net_device *dev)
{
        unsigned int key_len = tbl->key_len;
        u32 hash_val = pneigh_hash(pkey, key_len);

        return __pneigh_lookup_1(tbl->phash_buckets[hash_val],
                                 net, pkey, key_len, dev);
}
EXPORT_SYMBOL_GPL(__pneigh_lookup);

struct pneigh_entry * pneigh_lookup(struct neigh_table *tbl,
                                    struct net *net, const void *pkey,
                                    struct net_device *dev, int creat)
{
        struct pneigh_entry *n;
        unsigned int key_len = tbl->key_len;
        u32 hash_val = pneigh_hash(pkey, key_len);

        read_lock_bh(&tbl->lock);
        n = __pneigh_lookup_1(tbl->phash_buckets[hash_val],
                              net, pkey, key_len, dev);
        read_unlock_bh(&tbl->lock);

        if (n || !creat)
                goto out;

        ASSERT_RTNL();

        n = kzalloc(sizeof(*n) + key_len, GFP_KERNEL);
        if (!n)
                goto out;

        write_pnet(&n->net, net);
        memcpy(n->key, pkey, key_len);
        n->dev = dev;
        netdev_hold(dev, &n->dev_tracker, GFP_KERNEL);

        if (tbl->pconstructor && tbl->pconstructor(n)) {
                netdev_put(dev, &n->dev_tracker);
                kfree(n);
                n = NULL;
                goto out;
        }

        write_lock_bh(&tbl->lock);
        n->next = tbl->phash_buckets[hash_val];
        tbl->phash_buckets[hash_val] = n;
        write_unlock_bh(&tbl->lock);
out:
        return n;
}
EXPORT_SYMBOL(pneigh_lookup);


int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey,
                  struct net_device *dev)
{
        struct pneigh_entry *n, **np;
        unsigned int key_len = tbl->key_len;
        u32 hash_val = pneigh_hash(pkey, key_len);

        write_lock_bh(&tbl->lock);
        for (np = &tbl->phash_buckets[hash_val]; (n = *np) != NULL;
             np = &n->next) {
                if (!memcmp(n->key, pkey, key_len) && n->dev == dev &&
                    net_eq(pneigh_net(n), net)) {
                        *np = n->next;
                        write_unlock_bh(&tbl->lock);
                        if (tbl->pdestructor)
                                tbl->pdestructor(n);
                        netdev_put(n->dev, &n->dev_tracker);
                        kfree(n);
                        return 0;
                }
        }
        write_unlock_bh(&tbl->lock);
        return -ENOENT;
}

static int pneigh_ifdown_and_unlock(struct neigh_table *tbl,
                                    struct net_device *dev)
{
        struct pneigh_entry *n, **np, *freelist = NULL;
        u32 h;

        for (h = 0; h <= PNEIGH_HASHMASK; h++) {
                np = &tbl->phash_buckets[h];
                while ((n = *np) != NULL) {
                        if (!dev || n->dev == dev) {
                                *np = n->next;
                                n->next = freelist;
                                freelist = n;
                                continue;
                        }
                        np = &n->next;
                }
        }
        write_unlock_bh(&tbl->lock);
        while ((n = freelist)) {
                freelist = n->next;
                n->next = NULL;
                if (tbl->pdestructor)
                        tbl->pdestructor(n);
                netdev_put(n->dev, &n->dev_tracker);
                kfree(n);
        }
        return -ENOENT;
}

static inline void neigh_parms_put(struct neigh_parms *parms)
{
        if (refcount_dec_and_test(&parms->refcnt))
                kfree(parms);
}

/*
 *        neighbour must already be out of the table;
 *
 */
void neigh_destroy(struct neighbour *neigh)
{
        struct net_device *dev = neigh->dev;

        NEIGH_CACHE_STAT_INC(neigh->tbl, destroys);

        if (!neigh->dead) {
                pr_warn("Destroying alive neighbour %p\n", neigh);
                dump_stack();
                return;
        }

        if (neigh_del_timer(neigh))
                pr_warn("Impossible event\n");

        write_lock_bh(&neigh->lock);
        __skb_queue_purge(&neigh->arp_queue);
        write_unlock_bh(&neigh->lock);
        neigh->arp_queue_len_bytes = 0;

        if (dev->netdev_ops->ndo_neigh_destroy)
                dev->netdev_ops->ndo_neigh_destroy(dev, neigh);

        netdev_put(dev, &neigh->dev_tracker);
        neigh_parms_put(neigh->parms);

        neigh_dbg(2, "neigh %p is destroyed\n", neigh);

        atomic_dec(&neigh->tbl->entries);
        kfree_rcu(neigh, rcu);
}
EXPORT_SYMBOL(neigh_destroy);

/* Neighbour state is suspicious;
   disable fast path.

   Called with write_locked neigh.
 */
static void neigh_suspect(struct neighbour *neigh)
{
        neigh_dbg(2, "neigh %p is suspected\n", neigh);

        WRITE_ONCE(neigh->output, neigh->ops->output);
}

/* Neighbour state is OK;
   enable fast path.

   Called with write_locked neigh.
 */
static void neigh_connect(struct neighbour *neigh)
{
        neigh_dbg(2, "neigh %p is connected\n", neigh);

        WRITE_ONCE(neigh->output, neigh->ops->connected_output);
}

static void neigh_periodic_work(struct work_struct *work)
{
        struct neigh_table *tbl = container_of(work, struct neigh_table, gc_work.work);
        struct neigh_hash_table *nht;
        struct hlist_node *tmp;
        struct neighbour *n;
        unsigned int i;

        NEIGH_CACHE_STAT_INC(tbl, periodic_gc_runs);

        write_lock_bh(&tbl->lock);
        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));

        /*
         *        periodically recompute ReachableTime from random function
         */

        if (time_after(jiffies, tbl->last_rand + 300 * HZ)) {
                struct neigh_parms *p;

                WRITE_ONCE(tbl->last_rand, jiffies);
                list_for_each_entry(p, &tbl->parms_list, list)
                        p->reachable_time =
                                neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
        }

        if (atomic_read(&tbl->entries) < READ_ONCE(tbl->gc_thresh1))
                goto out;

        for (i = 0 ; i < (1 << nht->hash_shift); i++) {
                neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[i]) {
                        unsigned int state;

                        write_lock(&n->lock);

                        state = n->nud_state;
                        if ((state & (NUD_PERMANENT | NUD_IN_TIMER)) ||
                            (n->flags & NTF_EXT_LEARNED)) {
                                write_unlock(&n->lock);
                                continue;
                        }

                        if (time_before(n->used, n->confirmed) &&
                            time_is_before_eq_jiffies(n->confirmed))
                                n->used = n->confirmed;

                        if (refcount_read(&n->refcnt) == 1 &&
                            (state == NUD_FAILED ||
                             !time_in_range_open(jiffies, n->used,
                                                 n->used + NEIGH_VAR(n->parms, GC_STALETIME)))) {
                                hlist_del_rcu(&n->hash);
                                hlist_del_rcu(&n->dev_list);
                                neigh_mark_dead(n);
                                write_unlock(&n->lock);
                                neigh_cleanup_and_release(n);
                                continue;
                        }
                        write_unlock(&n->lock);
                }
                /*
                 * It's fine to release lock here, even if hash table
                 * grows while we are preempted.
                 */
                write_unlock_bh(&tbl->lock);
                cond_resched();
                write_lock_bh(&tbl->lock);
                nht = rcu_dereference_protected(tbl->nht,
                                                lockdep_is_held(&tbl->lock));
        }
out:
        /* Cycle through all hash buckets every BASE_REACHABLE_TIME/2 ticks.
         * ARP entry timeouts range from 1/2 BASE_REACHABLE_TIME to 3/2
         * BASE_REACHABLE_TIME.
         */
        queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
                              NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME) >> 1);
        write_unlock_bh(&tbl->lock);
}

static __inline__ int neigh_max_probes(struct neighbour *n)
{
        struct neigh_parms *p = n->parms;
        return NEIGH_VAR(p, UCAST_PROBES) + NEIGH_VAR(p, APP_PROBES) +
               (n->nud_state & NUD_PROBE ? NEIGH_VAR(p, MCAST_REPROBES) :
                NEIGH_VAR(p, MCAST_PROBES));
}

static void neigh_invalidate(struct neighbour *neigh)
        __releases(neigh->lock)
        __acquires(neigh->lock)
{
        struct sk_buff *skb;

        NEIGH_CACHE_STAT_INC(neigh->tbl, res_failed);
        neigh_dbg(2, "neigh %p is failed\n", neigh);
        neigh->updated = jiffies;

        /* It is very thin place. report_unreachable is very complicated
           routine. Particularly, it can hit the same neighbour entry!

           So that, we try to be accurate and avoid dead loop. --ANK
         */
        while (neigh->nud_state == NUD_FAILED &&
               (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
                write_unlock(&neigh->lock);
                neigh->ops->error_report(neigh, skb);
                write_lock(&neigh->lock);
        }
        __skb_queue_purge(&neigh->arp_queue);
        neigh->arp_queue_len_bytes = 0;
}

static void neigh_probe(struct neighbour *neigh)
        __releases(neigh->lock)
{
        struct sk_buff *skb = skb_peek_tail(&neigh->arp_queue);
        /* keep skb alive even if arp_queue overflows */
        if (skb)
                skb = skb_clone(skb, GFP_ATOMIC);
        write_unlock(&neigh->lock);
        if (neigh->ops->solicit)
                neigh->ops->solicit(neigh, skb);
        atomic_inc(&neigh->probes);
        consume_skb(skb);
}

/* Called when a timer expires for a neighbour entry. */

static void neigh_timer_handler(struct timer_list *t)
{
        unsigned long now, next;
        struct neighbour *neigh = timer_container_of(neigh, t, timer);
        unsigned int state;
        int notify = 0;

        write_lock(&neigh->lock);

        state = neigh->nud_state;
        now = jiffies;
        next = now + HZ;

        if (!(state & NUD_IN_TIMER))
                goto out;

        if (state & NUD_REACHABLE) {
                if (time_before_eq(now,
                                   neigh->confirmed + neigh->parms->reachable_time)) {
                        neigh_dbg(2, "neigh %p is still alive\n", neigh);
                        next = neigh->confirmed + neigh->parms->reachable_time;
                } else if (time_before_eq(now,
                                          neigh->used +
                                          NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is delayed\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        next = now + NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME);
                } else {
                        neigh_dbg(2, "neigh %p is suspected\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_STALE);
                        neigh->updated = jiffies;
                        neigh_suspect(neigh);
                        notify = 1;
                }
        } else if (state & NUD_DELAY) {
                if (time_before_eq(now,
                                   neigh->confirmed +
                                   NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME))) {
                        neigh_dbg(2, "neigh %p is now reachable\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_REACHABLE);
                        neigh->updated = jiffies;
                        neigh_connect(neigh);
                        notify = 1;
                        next = neigh->confirmed + neigh->parms->reachable_time;
                } else {
                        neigh_dbg(2, "neigh %p is probed\n", neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_PROBE);
                        neigh->updated = jiffies;
                        atomic_set(&neigh->probes, 0);
                        notify = 1;
                        next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
                                         HZ/100);
                }
        } else {
                /* NUD_PROBE|NUD_INCOMPLETE */
                next = now + max(NEIGH_VAR(neigh->parms, RETRANS_TIME), HZ/100);
        }

        if ((neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) &&
            atomic_read(&neigh->probes) >= neigh_max_probes(neigh)) {
                WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                notify = 1;
                neigh_invalidate(neigh);
                goto out;
        }

        if (neigh->nud_state & NUD_IN_TIMER) {
                if (time_before(next, jiffies + HZ/100))
                        next = jiffies + HZ/100;
                if (!mod_timer(&neigh->timer, next))
                        neigh_hold(neigh);
        }
        if (neigh->nud_state & (NUD_INCOMPLETE | NUD_PROBE)) {
                neigh_probe(neigh);
        } else {
out:
                write_unlock(&neigh->lock);
        }

        if (notify)
                neigh_update_notify(neigh, 0);

        trace_neigh_timer_handler(neigh, 0);

        neigh_release(neigh);
}

int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb,
                       const bool immediate_ok)
{
        int rc;
        bool immediate_probe = false;

        write_lock_bh(&neigh->lock);

        rc = 0;
        if (neigh->nud_state & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE))
                goto out_unlock_bh;
        if (neigh->dead)
                goto out_dead;

        if (!(neigh->nud_state & (NUD_STALE | NUD_INCOMPLETE))) {
                if (NEIGH_VAR(neigh->parms, MCAST_PROBES) +
                    NEIGH_VAR(neigh->parms, APP_PROBES)) {
                        unsigned long next, now = jiffies;

                        atomic_set(&neigh->probes,
                                   NEIGH_VAR(neigh->parms, UCAST_PROBES));
                        neigh_del_timer(neigh);
                        WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
                        neigh->updated = now;
                        if (!immediate_ok) {
                                next = now + 1;
                        } else {
                                immediate_probe = true;
                                next = now + max(NEIGH_VAR(neigh->parms,
                                                           RETRANS_TIME),
                                                 HZ / 100);
                        }
                        neigh_add_timer(neigh, next);
                } else {
                        WRITE_ONCE(neigh->nud_state, NUD_FAILED);
                        neigh->updated = jiffies;
                        write_unlock_bh(&neigh->lock);

                        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_FAILED);
                        return 1;
                }
        } else if (neigh->nud_state & NUD_STALE) {
                neigh_dbg(2, "neigh %p is delayed\n", neigh);
                neigh_del_timer(neigh);
                WRITE_ONCE(neigh->nud_state, NUD_DELAY);
                neigh->updated = jiffies;
                neigh_add_timer(neigh, jiffies +
                                NEIGH_VAR(neigh->parms, DELAY_PROBE_TIME));
        }

        if (neigh->nud_state == NUD_INCOMPLETE) {
                if (skb) {
                        while (neigh->arp_queue_len_bytes + skb->truesize >
                               NEIGH_VAR(neigh->parms, QUEUE_LEN_BYTES)) {
                                struct sk_buff *buff;

                                buff = __skb_dequeue(&neigh->arp_queue);
                                if (!buff)
                                        break;
                                neigh->arp_queue_len_bytes -= buff->truesize;
                                kfree_skb_reason(buff, SKB_DROP_REASON_NEIGH_QUEUEFULL);
                                NEIGH_CACHE_STAT_INC(neigh->tbl, unres_discards);
                        }
                        skb_dst_force(skb);
                        __skb_queue_tail(&neigh->arp_queue, skb);
                        neigh->arp_queue_len_bytes += skb->truesize;
                }
                rc = 1;
        }
out_unlock_bh:
        if (immediate_probe)
                neigh_probe(neigh);
        else
                write_unlock(&neigh->lock);
        local_bh_enable();
        trace_neigh_event_send_done(neigh, rc);
        return rc;

out_dead:
        if (neigh->nud_state & NUD_STALE)
                goto out_unlock_bh;
        write_unlock_bh(&neigh->lock);
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_DEAD);
        trace_neigh_event_send_dead(neigh, 1);
        return 1;
}
EXPORT_SYMBOL(__neigh_event_send);

static void neigh_update_hhs(struct neighbour *neigh)
{
        struct hh_cache *hh;
        void (*update)(struct hh_cache*, const struct net_device*, const unsigned char *)
                = NULL;

        if (neigh->dev->header_ops)
                update = neigh->dev->header_ops->cache_update;

        if (update) {
                hh = &neigh->hh;
                if (READ_ONCE(hh->hh_len)) {
                        write_seqlock_bh(&hh->hh_lock);
                        update(hh, neigh->dev, neigh->ha);
                        write_sequnlock_bh(&hh->hh_lock);
                }
        }
}

/* Generic update routine.
   -- lladdr is new lladdr or NULL, if it is not supplied.
   -- new    is new state.
   -- flags
        NEIGH_UPDATE_F_OVERRIDE allows to override existing lladdr,
                                if it is different.
        NEIGH_UPDATE_F_WEAK_OVERRIDE will suspect existing "connected"
                                lladdr instead of overriding it
                                if it is different.
        NEIGH_UPDATE_F_ADMIN        means that the change is administrative.
        NEIGH_UPDATE_F_USE        means that the entry is user triggered.
        NEIGH_UPDATE_F_MANAGED        means that the entry will be auto-refreshed.
        NEIGH_UPDATE_F_OVERRIDE_ISROUTER allows to override existing
                                NTF_ROUTER flag.
        NEIGH_UPDATE_F_ISROUTER        indicates if the neighbour is known as
                                a router.

   Caller MUST hold reference count on the entry.
 */
static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,
                          u8 new, u32 flags, u32 nlmsg_pid,
                          struct netlink_ext_ack *extack)
{
        bool gc_update = false, managed_update = false;
        int update_isrouter = 0;
        struct net_device *dev;
        int err, notify = 0;
        u8 old;

        trace_neigh_update(neigh, lladdr, new, flags, nlmsg_pid);

        write_lock_bh(&neigh->lock);

        dev    = neigh->dev;
        old    = neigh->nud_state;
        err    = -EPERM;

        if (neigh->dead) {
                NL_SET_ERR_MSG(extack, "Neighbor entry is now dead");
                new = old;
                goto out;
        }
        if (!(flags & NEIGH_UPDATE_F_ADMIN) &&
            (old & (NUD_NOARP | NUD_PERMANENT)))
                goto out;

        neigh_update_flags(neigh, flags, &notify, &gc_update, &managed_update);
        if (flags & (NEIGH_UPDATE_F_USE | NEIGH_UPDATE_F_MANAGED)) {
                new = old & ~NUD_PERMANENT;
                WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                goto out;
        }

        if (!(new & NUD_VALID)) {
                neigh_del_timer(neigh);
                if (old & NUD_CONNECTED)
                        neigh_suspect(neigh);
                WRITE_ONCE(neigh->nud_state, new);
                err = 0;
                notify = old & NUD_VALID;
                if ((old & (NUD_INCOMPLETE | NUD_PROBE)) &&
                    (new & NUD_FAILED)) {
                        neigh_invalidate(neigh);
                        notify = 1;
                }
                goto out;
        }

        /* Compare new lladdr with cached one */
        if (!dev->addr_len) {
                /* First case: device needs no address. */
                lladdr = neigh->ha;
        } else if (lladdr) {
                /* The second case: if something is already cached
                   and a new address is proposed:
                   - compare new & old
                   - if they are different, check override flag
                 */
                if ((old & NUD_VALID) &&
                    !memcmp(lladdr, neigh->ha, dev->addr_len))
                        lladdr = neigh->ha;
        } else {
                /* No address is supplied; if we know something,
                   use it, otherwise discard the request.
                 */
                err = -EINVAL;
                if (!(old & NUD_VALID)) {
                        NL_SET_ERR_MSG(extack, "No link layer address given");
                        goto out;
                }
                lladdr = neigh->ha;
        }

        /* Update confirmed timestamp for neighbour entry after we
         * received ARP packet even if it doesn't change IP to MAC binding.
         */
        if (new & NUD_CONNECTED)
                neigh->confirmed = jiffies;

        /* If entry was valid and address is not changed,
           do not change entry state, if new one is STALE.
         */
        err = 0;
        update_isrouter = flags & NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
        if (old & NUD_VALID) {
                if (lladdr != neigh->ha && !(flags & NEIGH_UPDATE_F_OVERRIDE)) {
                        update_isrouter = 0;
                        if ((flags & NEIGH_UPDATE_F_WEAK_OVERRIDE) &&
                            (old & NUD_CONNECTED)) {
                                lladdr = neigh->ha;
                                new = NUD_STALE;
                        } else
                                goto out;
                } else {
                        if (lladdr == neigh->ha && new == NUD_STALE &&
                            !(flags & NEIGH_UPDATE_F_ADMIN))
                                new = old;
                }
        }

        /* Update timestamp only once we know we will make a change to the
         * neighbour entry. Otherwise we risk to move the locktime window with
         * noop updates and ignore relevant ARP updates.
         */
        if (new != old || lladdr != neigh->ha)
                neigh->updated = jiffies;

        if (new != old) {
                neigh_del_timer(neigh);
                if (new & NUD_PROBE)
                        atomic_set(&neigh->probes, 0);
                if (new & NUD_IN_TIMER)
                        neigh_add_timer(neigh, (jiffies +
                                                ((new & NUD_REACHABLE) ?
                                                 neigh->parms->reachable_time :
                                                 0)));
                WRITE_ONCE(neigh->nud_state, new);
                notify = 1;
        }

        if (lladdr != neigh->ha) {
                write_seqlock(&neigh->ha_lock);
                memcpy(&neigh->ha, lladdr, dev->addr_len);
                write_sequnlock(&neigh->ha_lock);
                neigh_update_hhs(neigh);
                if (!(new & NUD_CONNECTED))
                        neigh->confirmed = jiffies -
                                      (NEIGH_VAR(neigh->parms, BASE_REACHABLE_TIME) << 1);
                notify = 1;
        }
        if (new == old)
                goto out;
        if (new & NUD_CONNECTED)
                neigh_connect(neigh);
        else
                neigh_suspect(neigh);
        if (!(old & NUD_VALID)) {
                struct sk_buff *skb;

                /* Again: avoid dead loop if something went wrong */

                while (neigh->nud_state & NUD_VALID &&
                       (skb = __skb_dequeue(&neigh->arp_queue)) != NULL) {
                        struct dst_entry *dst = skb_dst(skb);
                        struct neighbour *n2, *n1 = neigh;
                        write_unlock_bh(&neigh->lock);

                        rcu_read_lock();

                        /* Why not just use 'neigh' as-is?  The problem is that
                         * things such as shaper, eql, and sch_teql can end up
                         * using alternative, different, neigh objects to output
                         * the packet in the output path.  So what we need to do
                         * here is re-lookup the top-level neigh in the path so
                         * we can reinject the packet there.
                         */
                        n2 = NULL;
                        if (dst && dst->obsolete != DST_OBSOLETE_DEAD) {
                                n2 = dst_neigh_lookup_skb(dst, skb);
                                if (n2)
                                        n1 = n2;
                        }
                        READ_ONCE(n1->output)(n1, skb);
                        if (n2)
                                neigh_release(n2);
                        rcu_read_unlock();

                        write_lock_bh(&neigh->lock);
                }
                __skb_queue_purge(&neigh->arp_queue);
                neigh->arp_queue_len_bytes = 0;
        }
out:
        if (update_isrouter)
                neigh_update_is_router(neigh, flags, &notify);
        write_unlock_bh(&neigh->lock);
        if (((new ^ old) & NUD_PERMANENT) || gc_update)
                neigh_update_gc_list(neigh);
        if (managed_update)
                neigh_update_managed_list(neigh);
        if (notify)
                neigh_update_notify(neigh, nlmsg_pid);
        trace_neigh_update_done(neigh, err);
        return err;
}

int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,
                 u32 flags, u32 nlmsg_pid)
{
        return __neigh_update(neigh, lladdr, new, flags, nlmsg_pid, NULL);
}
EXPORT_SYMBOL(neigh_update);

/* Update the neigh to listen temporarily for probe responses, even if it is
 * in a NUD_FAILED state. The caller has to hold neigh->lock for writing.
 */
void __neigh_set_probe_once(struct neighbour *neigh)
{
        if (neigh->dead)
                return;
        neigh->updated = jiffies;
        if (!(neigh->nud_state & NUD_FAILED))
                return;
        WRITE_ONCE(neigh->nud_state, NUD_INCOMPLETE);
        atomic_set(&neigh->probes, neigh_max_probes(neigh));
        neigh_add_timer(neigh,
                        jiffies + max(NEIGH_VAR(neigh->parms, RETRANS_TIME),
                                      HZ/100));
}
EXPORT_SYMBOL(__neigh_set_probe_once);

struct neighbour *neigh_event_ns(struct neigh_table *tbl,
                                 u8 *lladdr, void *saddr,
                                 struct net_device *dev)
{
        struct neighbour *neigh = __neigh_lookup(tbl, saddr, dev,
                                                 lladdr || !dev->addr_len);
        if (neigh)
                neigh_update(neigh, lladdr, NUD_STALE,
                             NEIGH_UPDATE_F_OVERRIDE, 0);
        return neigh;
}
EXPORT_SYMBOL(neigh_event_ns);

/* called with read_lock_bh(&n->lock); */
static void neigh_hh_init(struct neighbour *n)
{
        struct net_device *dev = n->dev;
        __be16 prot = n->tbl->protocol;
        struct hh_cache        *hh = &n->hh;

        write_lock_bh(&n->lock);

        /* Only one thread can come in here and initialize the
         * hh_cache entry.
         */
        if (!hh->hh_len)
                dev->header_ops->cache(n, hh, prot);

        write_unlock_bh(&n->lock);
}

/* Slow and careful. */

int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb)
{
        int rc = 0;

        if (!neigh_event_send(neigh, skb)) {
                int err;
                struct net_device *dev = neigh->dev;
                unsigned int seq;

                if (dev->header_ops->cache && !READ_ONCE(neigh->hh.hh_len))
                        neigh_hh_init(neigh);

                do {
                        __skb_pull(skb, skb_network_offset(skb));
                        seq = read_seqbegin(&neigh->ha_lock);
                        err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                              neigh->ha, NULL, skb->len);
                } while (read_seqretry(&neigh->ha_lock, seq));

                if (err >= 0)
                        rc = dev_queue_xmit(skb);
                else
                        goto out_kfree_skb;
        }
out:
        return rc;
out_kfree_skb:
        rc = -EINVAL;
        kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_HH_FILLFAIL);
        goto out;
}
EXPORT_SYMBOL(neigh_resolve_output);

/* As fast as possible without hh cache */

int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb)
{
        struct net_device *dev = neigh->dev;
        unsigned int seq;
        int err;

        do {
                __skb_pull(skb, skb_network_offset(skb));
                seq = read_seqbegin(&neigh->ha_lock);
                err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                      neigh->ha, NULL, skb->len);
        } while (read_seqretry(&neigh->ha_lock, seq));

        if (err >= 0)
                err = dev_queue_xmit(skb);
        else {
                err = -EINVAL;
                kfree_skb_reason(skb, SKB_DROP_REASON_NEIGH_HH_FILLFAIL);
        }
        return err;
}
EXPORT_SYMBOL(neigh_connected_output);

int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb)
{
        return dev_queue_xmit(skb);
}
EXPORT_SYMBOL(neigh_direct_output);

static void neigh_managed_work(struct work_struct *work)
{
        struct neigh_table *tbl = container_of(work, struct neigh_table,
                                               managed_work.work);
        struct neighbour *neigh;

        write_lock_bh(&tbl->lock);
        list_for_each_entry(neigh, &tbl->managed_list, managed_list)
                neigh_event_send_probe(neigh, NULL, false);
        queue_delayed_work(system_power_efficient_wq, &tbl->managed_work,
                           NEIGH_VAR(&tbl->parms, INTERVAL_PROBE_TIME_MS));
        write_unlock_bh(&tbl->lock);
}

static void neigh_proxy_process(struct timer_list *t)
{
        struct neigh_table *tbl = timer_container_of(tbl, t, proxy_timer);
        long sched_next = 0;
        unsigned long now = jiffies;
        struct sk_buff *skb, *n;

        spin_lock(&tbl->proxy_queue.lock);

        skb_queue_walk_safe(&tbl->proxy_queue, skb, n) {
                long tdif = NEIGH_CB(skb)->sched_next - now;

                if (tdif <= 0) {
                        struct net_device *dev = skb->dev;

                        neigh_parms_qlen_dec(dev, tbl->family);
                        __skb_unlink(skb, &tbl->proxy_queue);

                        if (tbl->proxy_redo && netif_running(dev)) {
                                rcu_read_lock();
                                tbl->proxy_redo(skb);
                                rcu_read_unlock();
                        } else {
                                kfree_skb(skb);
                        }

                        dev_put(dev);
                } else if (!sched_next || tdif < sched_next)
                        sched_next = tdif;
        }
        timer_delete(&tbl->proxy_timer);
        if (sched_next)
                mod_timer(&tbl->proxy_timer, jiffies + sched_next);
        spin_unlock(&tbl->proxy_queue.lock);
}

static unsigned long neigh_proxy_delay(struct neigh_parms *p)
{
        /* If proxy_delay is zero, do not call get_random_u32_below()
         * as it is undefined behavior.
         */
        unsigned long proxy_delay = NEIGH_VAR(p, PROXY_DELAY);

        return proxy_delay ?
               jiffies + get_random_u32_below(proxy_delay) : jiffies;
}

void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p,
                    struct sk_buff *skb)
{
        unsigned long sched_next = neigh_proxy_delay(p);

        if (p->qlen > NEIGH_VAR(p, PROXY_QLEN)) {
                kfree_skb(skb);
                return;
        }

        NEIGH_CB(skb)->sched_next = sched_next;
        NEIGH_CB(skb)->flags |= LOCALLY_ENQUEUED;

        spin_lock(&tbl->proxy_queue.lock);
        if (timer_delete(&tbl->proxy_timer)) {
                if (time_before(tbl->proxy_timer.expires, sched_next))
                        sched_next = tbl->proxy_timer.expires;
        }
        skb_dst_drop(skb);
        dev_hold(skb->dev);
        __skb_queue_tail(&tbl->proxy_queue, skb);
        p->qlen++;
        mod_timer(&tbl->proxy_timer, sched_next);
        spin_unlock(&tbl->proxy_queue.lock);
}
EXPORT_SYMBOL(pneigh_enqueue);

static inline struct neigh_parms *lookup_neigh_parms(struct neigh_table *tbl,
                                                      struct net *net, int ifindex)
{
        struct neigh_parms *p;

        list_for_each_entry(p, &tbl->parms_list, list) {
                if ((p->dev && p->dev->ifindex == ifindex && net_eq(neigh_parms_net(p), net)) ||
                    (!p->dev && !ifindex && net_eq(net, &init_net)))
                        return p;
        }

        return NULL;
}

struct neigh_parms *neigh_parms_alloc(struct net_device *dev,
                                      struct neigh_table *tbl)
{
        struct neigh_parms *p;
        struct net *net = dev_net(dev);
        const struct net_device_ops *ops = dev->netdev_ops;

        p = kmemdup(&tbl->parms, sizeof(*p), GFP_KERNEL);
        if (p) {
                p->tbl                  = tbl;
                refcount_set(&p->refcnt, 1);
                p->reachable_time =
                                neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
                p->qlen = 0;
                netdev_hold(dev, &p->dev_tracker, GFP_KERNEL);
                p->dev = dev;
                write_pnet(&p->net, net);
                p->sysctl_table = NULL;

                if (ops->ndo_neigh_setup && ops->ndo_neigh_setup(dev, p)) {
                        netdev_put(dev, &p->dev_tracker);
                        kfree(p);
                        return NULL;
                }

                write_lock_bh(&tbl->lock);
                list_add(&p->list, &tbl->parms.list);
                write_unlock_bh(&tbl->lock);

                neigh_parms_data_state_cleanall(p);
        }
        return p;
}
EXPORT_SYMBOL(neigh_parms_alloc);

static void neigh_rcu_free_parms(struct rcu_head *head)
{
        struct neigh_parms *parms =
                container_of(head, struct neigh_parms, rcu_head);

        neigh_parms_put(parms);
}

void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms)
{
        if (!parms || parms == &tbl->parms)
                return;
        write_lock_bh(&tbl->lock);
        list_del(&parms->list);
        parms->dead = 1;
        write_unlock_bh(&tbl->lock);
        netdev_put(parms->dev, &parms->dev_tracker);
        call_rcu(&parms->rcu_head, neigh_rcu_free_parms);
}
EXPORT_SYMBOL(neigh_parms_release);

static struct lock_class_key neigh_table_proxy_queue_class;

static struct neigh_table __rcu *neigh_tables[NEIGH_NR_TABLES] __read_mostly;

void neigh_table_init(int index, struct neigh_table *tbl)
{
        unsigned long now = jiffies;
        unsigned long phsize;

        INIT_LIST_HEAD(&tbl->parms_list);
        INIT_LIST_HEAD(&tbl->gc_list);
        INIT_LIST_HEAD(&tbl->managed_list);

        list_add(&tbl->parms.list, &tbl->parms_list);
        write_pnet(&tbl->parms.net, &init_net);
        refcount_set(&tbl->parms.refcnt, 1);
        tbl->parms.reachable_time =
                          neigh_rand_reach_time(NEIGH_VAR(&tbl->parms, BASE_REACHABLE_TIME));
        tbl->parms.qlen = 0;

        tbl->stats = alloc_percpu(struct neigh_statistics);
        if (!tbl->stats)
                panic("cannot create neighbour cache statistics");

#ifdef CONFIG_PROC_FS
        if (!proc_create_seq_data(tbl->id, 0, init_net.proc_net_stat,
                              &neigh_stat_seq_ops, tbl))
                panic("cannot create neighbour proc dir entry");
#endif

        RCU_INIT_POINTER(tbl->nht, neigh_hash_alloc(3));

        phsize = (PNEIGH_HASHMASK + 1) * sizeof(struct pneigh_entry *);
        tbl->phash_buckets = kzalloc(phsize, GFP_KERNEL);

        if (!tbl->nht || !tbl->phash_buckets)
                panic("cannot allocate neighbour cache hashes");

        if (!tbl->entry_size)
                tbl->entry_size = ALIGN(offsetof(struct neighbour, primary_key) +
                                        tbl->key_len, NEIGH_PRIV_ALIGN);
        else
                WARN_ON(tbl->entry_size % NEIGH_PRIV_ALIGN);

        rwlock_init(&tbl->lock);

        INIT_DEFERRABLE_WORK(&tbl->gc_work, neigh_periodic_work);
        queue_delayed_work(system_power_efficient_wq, &tbl->gc_work,
                        tbl->parms.reachable_time);
        INIT_DEFERRABLE_WORK(&tbl->managed_work, neigh_managed_work);
        queue_delayed_work(system_power_efficient_wq, &tbl->managed_work, 0);

        timer_setup(&tbl->proxy_timer, neigh_proxy_process, 0);
        skb_queue_head_init_class(&tbl->proxy_queue,
                        &neigh_table_proxy_queue_class);

        tbl->last_flush = now;
        tbl->last_rand        = now + tbl->parms.reachable_time * 20;

        rcu_assign_pointer(neigh_tables[index], tbl);
}
EXPORT_SYMBOL(neigh_table_init);

/*
 * Only called from ndisc_cleanup(), which means this is dead code
 * because we no longer can unload IPv6 module.
 */
int neigh_table_clear(int index, struct neigh_table *tbl)
{
        RCU_INIT_POINTER(neigh_tables[index], NULL);
        synchronize_rcu();

        /* It is not clean... Fix it to unload IPv6 module safely */
        cancel_delayed_work_sync(&tbl->managed_work);
        cancel_delayed_work_sync(&tbl->gc_work);
        timer_delete_sync(&tbl->proxy_timer);
        pneigh_queue_purge(&tbl->proxy_queue, NULL, tbl->family);
        neigh_ifdown(tbl, NULL);
        if (atomic_read(&tbl->entries))
                pr_crit("neighbour leakage\n");

        call_rcu(&rcu_dereference_protected(tbl->nht, 1)->rcu,
                 neigh_hash_free_rcu);
        tbl->nht = NULL;

        kfree(tbl->phash_buckets);
        tbl->phash_buckets = NULL;

        remove_proc_entry(tbl->id, init_net.proc_net_stat);

        free_percpu(tbl->stats);
        tbl->stats = NULL;

        return 0;
}
EXPORT_SYMBOL(neigh_table_clear);

static struct neigh_table *neigh_find_table(int family)
{
        struct neigh_table *tbl = NULL;

        switch (family) {
        case AF_INET:
                tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ARP_TABLE]);
                break;
        case AF_INET6:
                tbl = rcu_dereference_rtnl(neigh_tables[NEIGH_ND_TABLE]);
                break;
        }

        return tbl;
}

const struct nla_policy nda_policy[NDA_MAX+1] = {
        [NDA_UNSPEC]                = { .strict_start_type = NDA_NH_ID },
        [NDA_DST]                = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [NDA_LLADDR]                = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },
        [NDA_CACHEINFO]                = { .len = sizeof(struct nda_cacheinfo) },
        [NDA_PROBES]                = { .type = NLA_U32 },
        [NDA_VLAN]                = { .type = NLA_U16 },
        [NDA_PORT]                = { .type = NLA_U16 },
        [NDA_VNI]                = { .type = NLA_U32 },
        [NDA_IFINDEX]                = { .type = NLA_U32 },
        [NDA_MASTER]                = { .type = NLA_U32 },
        [NDA_PROTOCOL]                = { .type = NLA_U8 },
        [NDA_NH_ID]                = { .type = NLA_U32 },
        [NDA_FLAGS_EXT]                = NLA_POLICY_MASK(NLA_U32, NTF_EXT_MASK),
        [NDA_FDB_EXT_ATTRS]        = { .type = NLA_NESTED },
};

static int neigh_delete(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct ndmsg *ndm;
        struct nlattr *dst_attr;
        struct neigh_table *tbl;
        struct neighbour *neigh;
        struct net_device *dev = NULL;
        int err = -EINVAL;

        ASSERT_RTNL();
        if (nlmsg_len(nlh) < sizeof(*ndm))
                goto out;

        dst_attr = nlmsg_find_attr(nlh, sizeof(*ndm), NDA_DST);
        if (!dst_attr) {
                NL_SET_ERR_MSG(extack, "Network address not specified");
                goto out;
        }

        ndm = nlmsg_data(nlh);
        if (ndm->ndm_ifindex) {
                dev = __dev_get_by_index(net, ndm->ndm_ifindex);
                if (dev == NULL) {
                        err = -ENODEV;
                        goto out;
                }
        }

        tbl = neigh_find_table(ndm->ndm_family);
        if (tbl == NULL)
                return -EAFNOSUPPORT;

        if (nla_len(dst_attr) < (int)tbl->key_len) {
                NL_SET_ERR_MSG(extack, "Invalid network address");
                goto out;
        }

        if (ndm->ndm_flags & NTF_PROXY) {
                err = pneigh_delete(tbl, net, nla_data(dst_attr), dev);
                goto out;
        }

        if (dev == NULL)
                goto out;

        neigh = neigh_lookup(tbl, nla_data(dst_attr), dev);
        if (neigh == NULL) {
                err = -ENOENT;
                goto out;
        }

        err = __neigh_update(neigh, NULL, NUD_FAILED,
                             NEIGH_UPDATE_F_OVERRIDE | NEIGH_UPDATE_F_ADMIN,
                             NETLINK_CB(skb).portid, extack);
        write_lock_bh(&tbl->lock);
        neigh_release(neigh);
        neigh_remove_one(neigh);
        write_unlock_bh(&tbl->lock);

out:
        return err;
}

static int neigh_add(struct sk_buff *skb, struct nlmsghdr *nlh,
                     struct netlink_ext_ack *extack)
{
        int flags = NEIGH_UPDATE_F_ADMIN | NEIGH_UPDATE_F_OVERRIDE |
                    NEIGH_UPDATE_F_OVERRIDE_ISROUTER;
        struct net *net = sock_net(skb->sk);
        struct ndmsg *ndm;
        struct nlattr *tb[NDA_MAX+1];
        struct neigh_table *tbl;
        struct net_device *dev = NULL;
        struct neighbour *neigh;
        void *dst, *lladdr;
        u8 protocol = 0;
        u32 ndm_flags;
        int err;

        ASSERT_RTNL();
        err = nlmsg_parse_deprecated(nlh, sizeof(*ndm), tb, NDA_MAX,
                                     nda_policy, extack);
        if (err < 0)
                goto out;

        err = -EINVAL;
        if (!tb[NDA_DST]) {
                NL_SET_ERR_MSG(extack, "Network address not specified");
                goto out;
        }

        ndm = nlmsg_data(nlh);
        ndm_flags = ndm->ndm_flags;
        if (tb[NDA_FLAGS_EXT]) {
                u32 ext = nla_get_u32(tb[NDA_FLAGS_EXT]);

                BUILD_BUG_ON(sizeof(neigh->flags) * BITS_PER_BYTE <
                             (sizeof(ndm->ndm_flags) * BITS_PER_BYTE +
                              hweight32(NTF_EXT_MASK)));
                ndm_flags |= (ext << NTF_EXT_SHIFT);
        }
        if (ndm->ndm_ifindex) {
                dev = __dev_get_by_index(net, ndm->ndm_ifindex);
                if (dev == NULL) {
                        err = -ENODEV;
                        goto out;
                }

                if (tb[NDA_LLADDR] && nla_len(tb[NDA_LLADDR]) < dev->addr_len) {
                        NL_SET_ERR_MSG(extack, "Invalid link address");
                        goto out;
                }
        }

        tbl = neigh_find_table(ndm->ndm_family);
        if (tbl == NULL)
                return -EAFNOSUPPORT;

        if (nla_len(tb[NDA_DST]) < (int)tbl->key_len) {
                NL_SET_ERR_MSG(extack, "Invalid network address");
                goto out;
        }

        dst = nla_data(tb[NDA_DST]);
        lladdr = tb[NDA_LLADDR] ? nla_data(tb[NDA_LLADDR]) : NULL;

        if (tb[NDA_PROTOCOL])
                protocol = nla_get_u8(tb[NDA_PROTOCOL]);
        if (ndm_flags & NTF_PROXY) {
                struct pneigh_entry *pn;

                if (ndm_flags & NTF_MANAGED) {
                        NL_SET_ERR_MSG(extack, "Invalid NTF_* flag combination");
                        goto out;
                }

                err = -ENOBUFS;
                pn = pneigh_lookup(tbl, net, dst, dev, 1);
                if (pn) {
                        pn->flags = ndm_flags;
                        if (protocol)
                                pn->protocol = protocol;
                        err = 0;
                }
                goto out;
        }

        if (!dev) {
                NL_SET_ERR_MSG(extack, "Device not specified");
                goto out;
        }

        if (tbl->allow_add && !tbl->allow_add(dev, extack)) {
                err = -EINVAL;
                goto out;
        }

        neigh = neigh_lookup(tbl, dst, dev);
        if (neigh == NULL) {
                bool ndm_permanent  = ndm->ndm_state & NUD_PERMANENT;
                bool exempt_from_gc = ndm_permanent ||
                                      ndm_flags & NTF_EXT_LEARNED;

                if (!(nlh->nlmsg_flags & NLM_F_CREATE)) {
                        err = -ENOENT;
                        goto out;
                }
                if (ndm_permanent && (ndm_flags & NTF_MANAGED)) {
                        NL_SET_ERR_MSG(extack, "Invalid NTF_* flag for permanent entry");
                        err = -EINVAL;
                        goto out;
                }

                neigh = ___neigh_create(tbl, dst, dev,
                                        ndm_flags &
                                        (NTF_EXT_LEARNED | NTF_MANAGED),
                                        exempt_from_gc, true);
                if (IS_ERR(neigh)) {
                        err = PTR_ERR(neigh);
                        goto out;
                }
        } else {
                if (nlh->nlmsg_flags & NLM_F_EXCL) {
                        err = -EEXIST;
                        neigh_release(neigh);
                        goto out;
                }

                if (!(nlh->nlmsg_flags & NLM_F_REPLACE))
                        flags &= ~(NEIGH_UPDATE_F_OVERRIDE |
                                   NEIGH_UPDATE_F_OVERRIDE_ISROUTER);
        }

        if (protocol)
                neigh->protocol = protocol;
        if (ndm_flags & NTF_EXT_LEARNED)
                flags |= NEIGH_UPDATE_F_EXT_LEARNED;
        if (ndm_flags & NTF_ROUTER)
                flags |= NEIGH_UPDATE_F_ISROUTER;
        if (ndm_flags & NTF_MANAGED)
                flags |= NEIGH_UPDATE_F_MANAGED;
        if (ndm_flags & NTF_USE)
                flags |= NEIGH_UPDATE_F_USE;

        err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,
                             NETLINK_CB(skb).portid, extack);
        if (!err && ndm_flags & (NTF_USE | NTF_MANAGED)) {
                neigh_event_send(neigh, NULL);
                err = 0;
        }
        neigh_release(neigh);
out:
        return err;
}

static int neightbl_fill_parms(struct sk_buff *skb, struct neigh_parms *parms)
{
        struct nlattr *nest;

        nest = nla_nest_start_noflag(skb, NDTA_PARMS);
        if (nest == NULL)
                return -ENOBUFS;

        if ((parms->dev &&
             nla_put_u32(skb, NDTPA_IFINDEX, parms->dev->ifindex)) ||
            nla_put_u32(skb, NDTPA_REFCNT, refcount_read(&parms->refcnt)) ||
            nla_put_u32(skb, NDTPA_QUEUE_LENBYTES,
                        NEIGH_VAR(parms, QUEUE_LEN_BYTES)) ||
            /* approximative value for deprecated QUEUE_LEN (in packets) */
            nla_put_u32(skb, NDTPA_QUEUE_LEN,
                        NEIGH_VAR(parms, QUEUE_LEN_BYTES) / SKB_TRUESIZE(ETH_FRAME_LEN)) ||
            nla_put_u32(skb, NDTPA_PROXY_QLEN, NEIGH_VAR(parms, PROXY_QLEN)) ||
            nla_put_u32(skb, NDTPA_APP_PROBES, NEIGH_VAR(parms, APP_PROBES)) ||
            nla_put_u32(skb, NDTPA_UCAST_PROBES,
                        NEIGH_VAR(parms, UCAST_PROBES)) ||
            nla_put_u32(skb, NDTPA_MCAST_PROBES,
                        NEIGH_VAR(parms, MCAST_PROBES)) ||
            nla_put_u32(skb, NDTPA_MCAST_REPROBES,
                        NEIGH_VAR(parms, MCAST_REPROBES)) ||
            nla_put_msecs(skb, NDTPA_REACHABLE_TIME, parms->reachable_time,
                          NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_BASE_REACHABLE_TIME,
                          NEIGH_VAR(parms, BASE_REACHABLE_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_GC_STALETIME,
                          NEIGH_VAR(parms, GC_STALETIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_DELAY_PROBE_TIME,
                          NEIGH_VAR(parms, DELAY_PROBE_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_RETRANS_TIME,
                          NEIGH_VAR(parms, RETRANS_TIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_ANYCAST_DELAY,
                          NEIGH_VAR(parms, ANYCAST_DELAY), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_PROXY_DELAY,
                          NEIGH_VAR(parms, PROXY_DELAY), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_LOCKTIME,
                          NEIGH_VAR(parms, LOCKTIME), NDTPA_PAD) ||
            nla_put_msecs(skb, NDTPA_INTERVAL_PROBE_TIME_MS,
                          NEIGH_VAR(parms, INTERVAL_PROBE_TIME_MS), NDTPA_PAD))
                goto nla_put_failure;
        return nla_nest_end(skb, nest);

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
                              u32 pid, u32 seq, int type, int flags)
{
        struct nlmsghdr *nlh;
        struct ndtmsg *ndtmsg;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ndtmsg = nlmsg_data(nlh);

        read_lock_bh(&tbl->lock);
        ndtmsg->ndtm_family = tbl->family;
        ndtmsg->ndtm_pad1   = 0;
        ndtmsg->ndtm_pad2   = 0;

        if (nla_put_string(skb, NDTA_NAME, tbl->id) ||
            nla_put_msecs(skb, NDTA_GC_INTERVAL, READ_ONCE(tbl->gc_interval),
                          NDTA_PAD) ||
            nla_put_u32(skb, NDTA_THRESH1, READ_ONCE(tbl->gc_thresh1)) ||
            nla_put_u32(skb, NDTA_THRESH2, READ_ONCE(tbl->gc_thresh2)) ||
            nla_put_u32(skb, NDTA_THRESH3, READ_ONCE(tbl->gc_thresh3)))
                goto nla_put_failure;
        {
                unsigned long now = jiffies;
                long flush_delta = now - READ_ONCE(tbl->last_flush);
                long rand_delta = now - READ_ONCE(tbl->last_rand);
                struct neigh_hash_table *nht;
                struct ndt_config ndc = {
                        .ndtc_key_len                = tbl->key_len,
                        .ndtc_entry_size        = tbl->entry_size,
                        .ndtc_entries                = atomic_read(&tbl->entries),
                        .ndtc_last_flush        = jiffies_to_msecs(flush_delta),
                        .ndtc_last_rand                = jiffies_to_msecs(rand_delta),
                        .ndtc_proxy_qlen        = READ_ONCE(tbl->proxy_queue.qlen),
                };

                rcu_read_lock();
                nht = rcu_dereference(tbl->nht);
                ndc.ndtc_hash_rnd = nht->hash_rnd[0];
                ndc.ndtc_hash_mask = ((1 << nht->hash_shift) - 1);
                rcu_read_unlock();

                if (nla_put(skb, NDTA_CONFIG, sizeof(ndc), &ndc))
                        goto nla_put_failure;
        }

        {
                int cpu;
                struct ndt_stats ndst;

                memset(&ndst, 0, sizeof(ndst));

                for_each_possible_cpu(cpu) {
                        struct neigh_statistics        *st;

                        st = per_cpu_ptr(tbl->stats, cpu);
                        ndst.ndts_allocs                += READ_ONCE(st->allocs);
                        ndst.ndts_destroys                += READ_ONCE(st->destroys);
                        ndst.ndts_hash_grows                += READ_ONCE(st->hash_grows);
                        ndst.ndts_res_failed                += READ_ONCE(st->res_failed);
                        ndst.ndts_lookups                += READ_ONCE(st->lookups);
                        ndst.ndts_hits                        += READ_ONCE(st->hits);
                        ndst.ndts_rcv_probes_mcast        += READ_ONCE(st->rcv_probes_mcast);
                        ndst.ndts_rcv_probes_ucast        += READ_ONCE(st->rcv_probes_ucast);
                        ndst.ndts_periodic_gc_runs        += READ_ONCE(st->periodic_gc_runs);
                        ndst.ndts_forced_gc_runs        += READ_ONCE(st->forced_gc_runs);
                        ndst.ndts_table_fulls                += READ_ONCE(st->table_fulls);
                }

                if (nla_put_64bit(skb, NDTA_STATS, sizeof(ndst), &ndst,
                                  NDTA_PAD))
                        goto nla_put_failure;
        }

        BUG_ON(tbl->parms.dev);
        if (neightbl_fill_parms(skb, &tbl->parms) < 0)
                goto nla_put_failure;

        read_unlock_bh(&tbl->lock);
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        read_unlock_bh(&tbl->lock);
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int neightbl_fill_param_info(struct sk_buff *skb,
                                    struct neigh_table *tbl,
                                    struct neigh_parms *parms,
                                    u32 pid, u32 seq, int type,
                                    unsigned int flags)
{
        struct ndtmsg *ndtmsg;
        struct nlmsghdr *nlh;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndtmsg), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        ndtmsg = nlmsg_data(nlh);

        read_lock_bh(&tbl->lock);
        ndtmsg->ndtm_family = tbl->family;
        ndtmsg->ndtm_pad1   = 0;
        ndtmsg->ndtm_pad2   = 0;

        if (nla_put_string(skb, NDTA_NAME, tbl->id) < 0 ||
            neightbl_fill_parms(skb, parms) < 0)
                goto errout;

        read_unlock_bh(&tbl->lock);
        nlmsg_end(skb, nlh);
        return 0;
errout:
        read_unlock_bh(&tbl->lock);
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static const struct nla_policy nl_neightbl_policy[NDTA_MAX+1] = {
        [NDTA_NAME]                = { .type = NLA_STRING },
        [NDTA_THRESH1]                = { .type = NLA_U32 },
        [NDTA_THRESH2]                = { .type = NLA_U32 },
        [NDTA_THRESH3]                = { .type = NLA_U32 },
        [NDTA_GC_INTERVAL]        = { .type = NLA_U64 },
        [NDTA_PARMS]                = { .type = NLA_NESTED },
};

static const struct nla_policy nl_ntbl_parm_policy[NDTPA_MAX+1] = {
        [NDTPA_IFINDEX]                        = { .type = NLA_U32 },
        [NDTPA_QUEUE_LEN]                = { .type = NLA_U32 },
        [NDTPA_QUEUE_LENBYTES]                = { .type = NLA_U32 },
        [NDTPA_PROXY_QLEN]                = { .type = NLA_U32 },
        [NDTPA_APP_PROBES]                = { .type = NLA_U32 },
        [NDTPA_UCAST_PROBES]                = { .type = NLA_U32 },
        [NDTPA_MCAST_PROBES]                = { .type = NLA_U32 },
        [NDTPA_MCAST_REPROBES]                = { .type = NLA_U32 },
        [NDTPA_BASE_REACHABLE_TIME]        = { .type = NLA_U64 },
        [NDTPA_GC_STALETIME]                = { .type = NLA_U64 },
        [NDTPA_DELAY_PROBE_TIME]        = { .type = NLA_U64 },
        [NDTPA_RETRANS_TIME]                = { .type = NLA_U64 },
        [NDTPA_ANYCAST_DELAY]                = { .type = NLA_U64 },
        [NDTPA_PROXY_DELAY]                = { .type = NLA_U64 },
        [NDTPA_LOCKTIME]                = { .type = NLA_U64 },
        [NDTPA_INTERVAL_PROBE_TIME_MS]        = { .type = NLA_U64, .min = 1 },
};

static int neightbl_set(struct sk_buff *skb, struct nlmsghdr *nlh,
                        struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(skb->sk);
        struct neigh_table *tbl;
        struct ndtmsg *ndtmsg;
        struct nlattr *tb[NDTA_MAX+1];
        bool found = false;
        int err, tidx;

        err = nlmsg_parse_deprecated(nlh, sizeof(*ndtmsg), tb, NDTA_MAX,
                                     nl_neightbl_policy, extack);
        if (err < 0)
                goto errout;

        if (tb[NDTA_NAME] == NULL) {
                err = -EINVAL;
                goto errout;
        }

        ndtmsg = nlmsg_data(nlh);

        for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
                tbl = rcu_dereference_rtnl(neigh_tables[tidx]);
                if (!tbl)
                        continue;
                if (ndtmsg->ndtm_family && tbl->family != ndtmsg->ndtm_family)
                        continue;
                if (nla_strcmp(tb[NDTA_NAME], tbl->id) == 0) {
                        found = true;
                        break;
                }
        }

        if (!found)
                return -ENOENT;

        /*
         * We acquire tbl->lock to be nice to the periodic timers and
         * make sure they always see a consistent set of values.
         */
        write_lock_bh(&tbl->lock);

        if (tb[NDTA_PARMS]) {
                struct nlattr *tbp[NDTPA_MAX+1];
                struct neigh_parms *p;
                int i, ifindex = 0;

                err = nla_parse_nested_deprecated(tbp, NDTPA_MAX,
                                                  tb[NDTA_PARMS],
                                                  nl_ntbl_parm_policy, extack);
                if (err < 0)
                        goto errout_tbl_lock;

                if (tbp[NDTPA_IFINDEX])
                        ifindex = nla_get_u32(tbp[NDTPA_IFINDEX]);

                p = lookup_neigh_parms(tbl, net, ifindex);
                if (p == NULL) {
                        err = -ENOENT;
                        goto errout_tbl_lock;
                }

                for (i = 1; i <= NDTPA_MAX; i++) {
                        if (tbp[i] == NULL)
                                continue;

                        switch (i) {
                        case NDTPA_QUEUE_LEN:
                                NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
                                              nla_get_u32(tbp[i]) *
                                              SKB_TRUESIZE(ETH_FRAME_LEN));
                                break;
                        case NDTPA_QUEUE_LENBYTES:
                                NEIGH_VAR_SET(p, QUEUE_LEN_BYTES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_PROXY_QLEN:
                                NEIGH_VAR_SET(p, PROXY_QLEN,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_APP_PROBES:
                                NEIGH_VAR_SET(p, APP_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_UCAST_PROBES:
                                NEIGH_VAR_SET(p, UCAST_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_MCAST_PROBES:
                                NEIGH_VAR_SET(p, MCAST_PROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_MCAST_REPROBES:
                                NEIGH_VAR_SET(p, MCAST_REPROBES,
                                              nla_get_u32(tbp[i]));
                                break;
                        case NDTPA_BASE_REACHABLE_TIME:
                                NEIGH_VAR_SET(p, BASE_REACHABLE_TIME,
                                              nla_get_msecs(tbp[i]));
                                /* update reachable_time as well, otherwise, the change will
                                 * only be effective after the next time neigh_periodic_work
                                 * decides to recompute it (can be multiple minutes)
                                 */
                                p->reachable_time =
                                        neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
                                break;
                        case NDTPA_GC_STALETIME:
                                NEIGH_VAR_SET(p, GC_STALETIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_DELAY_PROBE_TIME:
                                NEIGH_VAR_SET(p, DELAY_PROBE_TIME,
                                              nla_get_msecs(tbp[i]));
                                call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
                                break;
                        case NDTPA_INTERVAL_PROBE_TIME_MS:
                                NEIGH_VAR_SET(p, INTERVAL_PROBE_TIME_MS,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_RETRANS_TIME:
                                NEIGH_VAR_SET(p, RETRANS_TIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_ANYCAST_DELAY:
                                NEIGH_VAR_SET(p, ANYCAST_DELAY,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_PROXY_DELAY:
                                NEIGH_VAR_SET(p, PROXY_DELAY,
                                              nla_get_msecs(tbp[i]));
                                break;
                        case NDTPA_LOCKTIME:
                                NEIGH_VAR_SET(p, LOCKTIME,
                                              nla_get_msecs(tbp[i]));
                                break;
                        }
                }
        }

        err = -ENOENT;
        if ((tb[NDTA_THRESH1] || tb[NDTA_THRESH2] ||
             tb[NDTA_THRESH3] || tb[NDTA_GC_INTERVAL]) &&
            !net_eq(net, &init_net))
                goto errout_tbl_lock;

        if (tb[NDTA_THRESH1])
                WRITE_ONCE(tbl->gc_thresh1, nla_get_u32(tb[NDTA_THRESH1]));

        if (tb[NDTA_THRESH2])
                WRITE_ONCE(tbl->gc_thresh2, nla_get_u32(tb[NDTA_THRESH2]));

        if (tb[NDTA_THRESH3])
                WRITE_ONCE(tbl->gc_thresh3, nla_get_u32(tb[NDTA_THRESH3]));

        if (tb[NDTA_GC_INTERVAL])
                WRITE_ONCE(tbl->gc_interval, nla_get_msecs(tb[NDTA_GC_INTERVAL]));

        err = 0;

errout_tbl_lock:
        write_unlock_bh(&tbl->lock);
errout:
        return err;
}

static int neightbl_valid_dump_info(const struct nlmsghdr *nlh,
                                    struct netlink_ext_ack *extack)
{
        struct ndtmsg *ndtm;

        ndtm = nlmsg_payload(nlh, sizeof(*ndtm));
        if (!ndtm) {
                NL_SET_ERR_MSG(extack, "Invalid header for neighbor table dump request");
                return -EINVAL;
        }

        if (ndtm->ndtm_pad1  || ndtm->ndtm_pad2) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor table dump request");
                return -EINVAL;
        }

        if (nlmsg_attrlen(nlh, sizeof(*ndtm))) {
                NL_SET_ERR_MSG(extack, "Invalid data after header in neighbor table dump request");
                return -EINVAL;
        }

        return 0;
}

static int neightbl_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct net *net = sock_net(skb->sk);
        int family, tidx, nidx = 0;
        int tbl_skip = cb->args[0];
        int neigh_skip = cb->args[1];
        struct neigh_table *tbl;

        if (cb->strict_check) {
                int err = neightbl_valid_dump_info(nlh, cb->extack);

                if (err < 0)
                        return err;
        }

        family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;

        for (tidx = 0; tidx < NEIGH_NR_TABLES; tidx++) {
                struct neigh_parms *p;

                tbl = rcu_dereference_rtnl(neigh_tables[tidx]);
                if (!tbl)
                        continue;

                if (tidx < tbl_skip || (family && tbl->family != family))
                        continue;

                if (neightbl_fill_info(skb, tbl, NETLINK_CB(cb->skb).portid,
                                       nlh->nlmsg_seq, RTM_NEWNEIGHTBL,
                                       NLM_F_MULTI) < 0)
                        break;

                nidx = 0;
                p = list_next_entry(&tbl->parms, list);
                list_for_each_entry_from(p, &tbl->parms_list, list) {
                        if (!net_eq(neigh_parms_net(p), net))
                                continue;

                        if (nidx < neigh_skip)
                                goto next;

                        if (neightbl_fill_param_info(skb, tbl, p,
                                                     NETLINK_CB(cb->skb).portid,
                                                     nlh->nlmsg_seq,
                                                     RTM_NEWNEIGHTBL,
                                                     NLM_F_MULTI) < 0)
                                goto out;
                next:
                        nidx++;
                }

                neigh_skip = 0;
        }
out:
        cb->args[0] = tidx;
        cb->args[1] = nidx;

        return skb->len;
}

static int neigh_fill_info(struct sk_buff *skb, struct neighbour *neigh,
                           u32 pid, u32 seq, int type, unsigned int flags)
{
        u32 neigh_flags, neigh_flags_ext;
        unsigned long now = jiffies;
        struct nda_cacheinfo ci;
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        neigh_flags_ext = neigh->flags >> NTF_EXT_SHIFT;
        neigh_flags     = neigh->flags & NTF_OLD_MASK;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family         = neigh->ops->family;
        ndm->ndm_pad1    = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags         = neigh_flags;
        ndm->ndm_type         = neigh->type;
        ndm->ndm_ifindex = neigh->dev->ifindex;

        if (nla_put(skb, NDA_DST, neigh->tbl->key_len, neigh->primary_key))
                goto nla_put_failure;

        read_lock_bh(&neigh->lock);
        ndm->ndm_state         = neigh->nud_state;
        if (neigh->nud_state & NUD_VALID) {
                char haddr[MAX_ADDR_LEN];

                neigh_ha_snapshot(haddr, neigh, neigh->dev);
                if (nla_put(skb, NDA_LLADDR, neigh->dev->addr_len, haddr) < 0) {
                        read_unlock_bh(&neigh->lock);
                        goto nla_put_failure;
                }
        }

        ci.ndm_used         = jiffies_to_clock_t(now - neigh->used);
        ci.ndm_confirmed = jiffies_to_clock_t(now - neigh->confirmed);
        ci.ndm_updated         = jiffies_to_clock_t(now - neigh->updated);
        ci.ndm_refcnt         = refcount_read(&neigh->refcnt) - 1;
        read_unlock_bh(&neigh->lock);

        if (nla_put_u32(skb, NDA_PROBES, atomic_read(&neigh->probes)) ||
            nla_put(skb, NDA_CACHEINFO, sizeof(ci), &ci))
                goto nla_put_failure;

        if (neigh->protocol && nla_put_u8(skb, NDA_PROTOCOL, neigh->protocol))
                goto nla_put_failure;
        if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static int pneigh_fill_info(struct sk_buff *skb, struct pneigh_entry *pn,
                            u32 pid, u32 seq, int type, unsigned int flags,
                            struct neigh_table *tbl)
{
        u32 neigh_flags, neigh_flags_ext;
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;

        nlh = nlmsg_put(skb, pid, seq, type, sizeof(*ndm), flags);
        if (nlh == NULL)
                return -EMSGSIZE;

        neigh_flags_ext = pn->flags >> NTF_EXT_SHIFT;
        neigh_flags     = pn->flags & NTF_OLD_MASK;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family         = tbl->family;
        ndm->ndm_pad1    = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags         = neigh_flags | NTF_PROXY;
        ndm->ndm_type         = RTN_UNICAST;
        ndm->ndm_ifindex = pn->dev ? pn->dev->ifindex : 0;
        ndm->ndm_state         = NUD_NONE;

        if (nla_put(skb, NDA_DST, tbl->key_len, pn->key))
                goto nla_put_failure;

        if (pn->protocol && nla_put_u8(skb, NDA_PROTOCOL, pn->protocol))
                goto nla_put_failure;
        if (neigh_flags_ext && nla_put_u32(skb, NDA_FLAGS_EXT, neigh_flags_ext))
                goto nla_put_failure;

        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static void neigh_update_notify(struct neighbour *neigh, u32 nlmsg_pid)
{
        call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, neigh);
        __neigh_notify(neigh, RTM_NEWNEIGH, 0, nlmsg_pid);
}

static bool neigh_master_filtered(struct net_device *dev, int master_idx)
{
        struct net_device *master;

        if (!master_idx)
                return false;

        master = dev ? netdev_master_upper_dev_get_rcu(dev) : NULL;

        /* 0 is already used to denote NDA_MASTER wasn't passed, therefore need another
         * invalid value for ifindex to denote "no master".
         */
        if (master_idx == -1)
                return !!master;

        if (!master || master->ifindex != master_idx)
                return true;

        return false;
}

static bool neigh_ifindex_filtered(struct net_device *dev, int filter_idx)
{
        if (filter_idx && (!dev || dev->ifindex != filter_idx))
                return true;

        return false;
}

struct neigh_dump_filter {
        int master_idx;
        int dev_idx;
};

static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
                            struct netlink_callback *cb,
                            struct neigh_dump_filter *filter)
{
        struct net *net = sock_net(skb->sk);
        struct neighbour *n;
        int err = 0, h, s_h = cb->args[1];
        int idx, s_idx = idx = cb->args[2];
        struct neigh_hash_table *nht;
        unsigned int flags = NLM_F_MULTI;

        if (filter->dev_idx || filter->master_idx)
                flags |= NLM_F_DUMP_FILTERED;

        nht = rcu_dereference(tbl->nht);

        for (h = s_h; h < (1 << nht->hash_shift); h++) {
                if (h > s_h)
                        s_idx = 0;
                idx = 0;
                neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[h]) {
                        if (idx < s_idx || !net_eq(dev_net(n->dev), net))
                                goto next;
                        if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
                            neigh_master_filtered(n->dev, filter->master_idx))
                                goto next;
                        err = neigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
                                              cb->nlh->nlmsg_seq,
                                              RTM_NEWNEIGH, flags);
                        if (err < 0)
                                goto out;
next:
                        idx++;
                }
        }
out:
        cb->args[1] = h;
        cb->args[2] = idx;
        return err;
}

static int pneigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
                             struct netlink_callback *cb,
                             struct neigh_dump_filter *filter)
{
        struct pneigh_entry *n;
        struct net *net = sock_net(skb->sk);
        int err = 0, h, s_h = cb->args[3];
        int idx, s_idx = idx = cb->args[4];
        unsigned int flags = NLM_F_MULTI;

        if (filter->dev_idx || filter->master_idx)
                flags |= NLM_F_DUMP_FILTERED;

        read_lock_bh(&tbl->lock);

        for (h = s_h; h <= PNEIGH_HASHMASK; h++) {
                if (h > s_h)
                        s_idx = 0;
                for (n = tbl->phash_buckets[h], idx = 0; n; n = n->next) {
                        if (idx < s_idx || pneigh_net(n) != net)
                                goto next;
                        if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
                            neigh_master_filtered(n->dev, filter->master_idx))
                                goto next;
                        err = pneigh_fill_info(skb, n, NETLINK_CB(cb->skb).portid,
                                               cb->nlh->nlmsg_seq,
                                               RTM_NEWNEIGH, flags, tbl);
                        if (err < 0) {
                                read_unlock_bh(&tbl->lock);
                                goto out;
                        }
                next:
                        idx++;
                }
        }

        read_unlock_bh(&tbl->lock);
out:
        cb->args[3] = h;
        cb->args[4] = idx;
        return err;
}

static int neigh_valid_dump_req(const struct nlmsghdr *nlh,
                                bool strict_check,
                                struct neigh_dump_filter *filter,
                                struct netlink_ext_ack *extack)
{
        struct nlattr *tb[NDA_MAX + 1];
        int err, i;

        if (strict_check) {
                struct ndmsg *ndm;

                ndm = nlmsg_payload(nlh, sizeof(*ndm));
                if (!ndm) {
                        NL_SET_ERR_MSG(extack, "Invalid header for neighbor dump request");
                        return -EINVAL;
                }

                if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_ifindex ||
                    ndm->ndm_state || ndm->ndm_type) {
                        NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor dump request");
                        return -EINVAL;
                }

                if (ndm->ndm_flags & ~NTF_PROXY) {
                        NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor dump request");
                        return -EINVAL;
                }

                err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg),
                                                    tb, NDA_MAX, nda_policy,
                                                    extack);
        } else {
                err = nlmsg_parse_deprecated(nlh, sizeof(struct ndmsg), tb,
                                             NDA_MAX, nda_policy, extack);
        }
        if (err < 0)
                return err;

        for (i = 0; i <= NDA_MAX; ++i) {
                if (!tb[i])
                        continue;

                /* all new attributes should require strict_check */
                switch (i) {
                case NDA_IFINDEX:
                        filter->dev_idx = nla_get_u32(tb[i]);
                        break;
                case NDA_MASTER:
                        filter->master_idx = nla_get_u32(tb[i]);
                        break;
                default:
                        if (strict_check) {
                                NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor dump request");
                                return -EINVAL;
                        }
                }
        }

        return 0;
}

static int neigh_dump_info(struct sk_buff *skb, struct netlink_callback *cb)
{
        const struct nlmsghdr *nlh = cb->nlh;
        struct neigh_dump_filter filter = {};
        struct neigh_table *tbl;
        int t, family, s_t;
        int proxy = 0;
        int err;

        family = ((struct rtgenmsg *)nlmsg_data(nlh))->rtgen_family;

        /* check for full ndmsg structure presence, family member is
         * the same for both structures
         */
        if (nlmsg_len(nlh) >= sizeof(struct ndmsg) &&
            ((struct ndmsg *)nlmsg_data(nlh))->ndm_flags == NTF_PROXY)
                proxy = 1;

        err = neigh_valid_dump_req(nlh, cb->strict_check, &filter, cb->extack);
        if (err < 0 && cb->strict_check)
                return err;
        err = 0;

        s_t = cb->args[0];

        rcu_read_lock();
        for (t = 0; t < NEIGH_NR_TABLES; t++) {
                tbl = rcu_dereference(neigh_tables[t]);

                if (!tbl)
                        continue;
                if (t < s_t || (family && tbl->family != family))
                        continue;
                if (t > s_t)
                        memset(&cb->args[1], 0, sizeof(cb->args) -
                                                sizeof(cb->args[0]));
                if (proxy)
                        err = pneigh_dump_table(tbl, skb, cb, &filter);
                else
                        err = neigh_dump_table(tbl, skb, cb, &filter);
                if (err < 0)
                        break;
        }
        rcu_read_unlock();

        cb->args[0] = t;
        return err;
}

static int neigh_valid_get_req(const struct nlmsghdr *nlh,
                               struct neigh_table **tbl,
                               void **dst, int *dev_idx, u8 *ndm_flags,
                               struct netlink_ext_ack *extack)
{
        struct nlattr *tb[NDA_MAX + 1];
        struct ndmsg *ndm;
        int err, i;

        ndm = nlmsg_payload(nlh, sizeof(*ndm));
        if (!ndm) {
                NL_SET_ERR_MSG(extack, "Invalid header for neighbor get request");
                return -EINVAL;
        }

        if (ndm->ndm_pad1  || ndm->ndm_pad2  || ndm->ndm_state ||
            ndm->ndm_type) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor get request");
                return -EINVAL;
        }

        if (ndm->ndm_flags & ~NTF_PROXY) {
                NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor get request");
                return -EINVAL;
        }

        err = nlmsg_parse_deprecated_strict(nlh, sizeof(struct ndmsg), tb,
                                            NDA_MAX, nda_policy, extack);
        if (err < 0)
                return err;

        *ndm_flags = ndm->ndm_flags;
        *dev_idx = ndm->ndm_ifindex;
        *tbl = neigh_find_table(ndm->ndm_family);
        if (*tbl == NULL) {
                NL_SET_ERR_MSG(extack, "Unsupported family in header for neighbor get request");
                return -EAFNOSUPPORT;
        }

        for (i = 0; i <= NDA_MAX; ++i) {
                if (!tb[i])
                        continue;

                switch (i) {
                case NDA_DST:
                        if (nla_len(tb[i]) != (int)(*tbl)->key_len) {
                                NL_SET_ERR_MSG(extack, "Invalid network address in neighbor get request");
                                return -EINVAL;
                        }
                        *dst = nla_data(tb[i]);
                        break;
                default:
                        NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor get request");
                        return -EINVAL;
                }
        }

        return 0;
}

static inline size_t neigh_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg))
               + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
               + nla_total_size(MAX_ADDR_LEN) /* NDA_LLADDR */
               + nla_total_size(sizeof(struct nda_cacheinfo))
               + nla_total_size(4)  /* NDA_PROBES */
               + nla_total_size(4)  /* NDA_FLAGS_EXT */
               + nla_total_size(1); /* NDA_PROTOCOL */
}

static int neigh_get_reply(struct net *net, struct neighbour *neigh,
                           u32 pid, u32 seq)
{
        struct sk_buff *skb;
        int err = 0;

        skb = nlmsg_new(neigh_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        err = neigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0);
        if (err) {
                kfree_skb(skb);
                goto errout;
        }

        err = rtnl_unicast(skb, net, pid);
errout:
        return err;
}

static inline size_t pneigh_nlmsg_size(void)
{
        return NLMSG_ALIGN(sizeof(struct ndmsg))
               + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */
               + nla_total_size(4)  /* NDA_FLAGS_EXT */
               + nla_total_size(1); /* NDA_PROTOCOL */
}

static int pneigh_get_reply(struct net *net, struct pneigh_entry *neigh,
                            u32 pid, u32 seq, struct neigh_table *tbl)
{
        struct sk_buff *skb;
        int err = 0;

        skb = nlmsg_new(pneigh_nlmsg_size(), GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        err = pneigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0, tbl);
        if (err) {
                kfree_skb(skb);
                goto errout;
        }

        err = rtnl_unicast(skb, net, pid);
errout:
        return err;
}

static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                     struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct net_device *dev = NULL;
        struct neigh_table *tbl = NULL;
        struct neighbour *neigh;
        void *dst = NULL;
        u8 ndm_flags = 0;
        int dev_idx = 0;
        int err;

        err = neigh_valid_get_req(nlh, &tbl, &dst, &dev_idx, &ndm_flags,
                                  extack);
        if (err < 0)
                return err;

        if (dev_idx) {
                dev = __dev_get_by_index(net, dev_idx);
                if (!dev) {
                        NL_SET_ERR_MSG(extack, "Unknown device ifindex");
                        return -ENODEV;
                }
        }

        if (!dst) {
                NL_SET_ERR_MSG(extack, "Network address not specified");
                return -EINVAL;
        }

        if (ndm_flags & NTF_PROXY) {
                struct pneigh_entry *pn;

                pn = pneigh_lookup(tbl, net, dst, dev, 0);
                if (!pn) {
                        NL_SET_ERR_MSG(extack, "Proxy neighbour entry not found");
                        return -ENOENT;
                }
                return pneigh_get_reply(net, pn, NETLINK_CB(in_skb).portid,
                                        nlh->nlmsg_seq, tbl);
        }

        if (!dev) {
                NL_SET_ERR_MSG(extack, "No device specified");
                return -EINVAL;
        }

        neigh = neigh_lookup(tbl, dst, dev);
        if (!neigh) {
                NL_SET_ERR_MSG(extack, "Neighbour entry not found");
                return -ENOENT;
        }

        err = neigh_get_reply(net, neigh, NETLINK_CB(in_skb).portid,
                              nlh->nlmsg_seq);

        neigh_release(neigh);

        return err;
}

void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void *), void *cookie)
{
        int chain;
        struct neigh_hash_table *nht;

        rcu_read_lock();
        nht = rcu_dereference(tbl->nht);

        read_lock_bh(&tbl->lock); /* avoid resizes */
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
                struct neighbour *n;

                neigh_for_each_in_bucket(n, &nht->hash_heads[chain])
                        cb(n, cookie);
        }
        read_unlock_bh(&tbl->lock);
        rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_for_each);

/* The tbl->lock must be held as a writer and BH disabled. */
void __neigh_for_each_release(struct neigh_table *tbl,
                              int (*cb)(struct neighbour *))
{
        struct neigh_hash_table *nht;
        int chain;

        nht = rcu_dereference_protected(tbl->nht,
                                        lockdep_is_held(&tbl->lock));
        for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
                struct hlist_node *tmp;
                struct neighbour *n;

                neigh_for_each_in_bucket_safe(n, tmp, &nht->hash_heads[chain]) {
                        int release;

                        write_lock(&n->lock);
                        release = cb(n);
                        if (release) {
                                hlist_del_rcu(&n->hash);
                                hlist_del_rcu(&n->dev_list);
                                neigh_mark_dead(n);
                        }
                        write_unlock(&n->lock);
                        if (release)
                                neigh_cleanup_and_release(n);
                }
        }
}
EXPORT_SYMBOL(__neigh_for_each_release);

int neigh_xmit(int index, struct net_device *dev,
               const void *addr, struct sk_buff *skb)
{
        int err = -EAFNOSUPPORT;

        if (likely(index < NEIGH_NR_TABLES)) {
                struct neigh_table *tbl;
                struct neighbour *neigh;

                rcu_read_lock();
                tbl = rcu_dereference(neigh_tables[index]);
                if (!tbl)
                        goto out_unlock;
                if (index == NEIGH_ARP_TABLE) {
                        u32 key = *((u32 *)addr);

                        neigh = __ipv4_neigh_lookup_noref(dev, key);
                } else {
                        neigh = __neigh_lookup_noref(tbl, addr, dev);
                }
                if (!neigh)
                        neigh = __neigh_create(tbl, addr, dev, false);
                err = PTR_ERR(neigh);
                if (IS_ERR(neigh)) {
                        rcu_read_unlock();
                        goto out_kfree_skb;
                }
                err = READ_ONCE(neigh->output)(neigh, skb);
out_unlock:
                rcu_read_unlock();
        }
        else if (index == NEIGH_LINK_TABLE) {
                err = dev_hard_header(skb, dev, ntohs(skb->protocol),
                                      addr, NULL, skb->len);
                if (err < 0)
                        goto out_kfree_skb;
                err = dev_queue_xmit(skb);
        }
out:
        return err;
out_kfree_skb:
        kfree_skb(skb);
        goto out;
}
EXPORT_SYMBOL(neigh_xmit);

#ifdef CONFIG_PROC_FS

static struct neighbour *neigh_get_valid(struct seq_file *seq,
                                         struct neighbour *n,
                                         loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);

        if (!net_eq(dev_net(n->dev), net))
                return NULL;

        if (state->neigh_sub_iter) {
                loff_t fakep = 0;
                void *v;

                v = state->neigh_sub_iter(state, n, pos ? pos : &fakep);
                if (!v)
                        return NULL;
                if (pos)
                        return v;
        }

        if (!(state->flags & NEIGH_SEQ_SKIP_NOARP))
                return n;

        if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
                return n;

        return NULL;
}

static struct neighbour *neigh_get_first(struct seq_file *seq)
{
        struct neigh_seq_state *state = seq->private;
        struct neigh_hash_table *nht = state->nht;
        struct neighbour *n, *tmp;

        state->flags &= ~NEIGH_SEQ_IS_PNEIGH;

        while (++state->bucket < (1 << nht->hash_shift)) {
                neigh_for_each_in_bucket(n, &nht->hash_heads[state->bucket]) {
                        tmp = neigh_get_valid(seq, n, NULL);
                        if (tmp)
                                return tmp;
                }
        }

        return NULL;
}

static struct neighbour *neigh_get_next(struct seq_file *seq,
                                        struct neighbour *n,
                                        loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct neighbour *tmp;

        if (state->neigh_sub_iter) {
                void *v = state->neigh_sub_iter(state, n, pos);

                if (v)
                        return n;
        }

        hlist_for_each_entry_continue(n, hash) {
                tmp = neigh_get_valid(seq, n, pos);
                if (tmp) {
                        n = tmp;
                        goto out;
                }
        }

        n = neigh_get_first(seq);
out:
        if (n && pos)
                --(*pos);

        return n;
}

static struct neighbour *neigh_get_idx(struct seq_file *seq, loff_t *pos)
{
        struct neighbour *n = neigh_get_first(seq);

        if (n) {
                --(*pos);
                while (*pos) {
                        n = neigh_get_next(seq, n, pos);
                        if (!n)
                                break;
                }
        }
        return *pos ? NULL : n;
}

static struct pneigh_entry *pneigh_get_first(struct seq_file *seq)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);
        struct neigh_table *tbl = state->tbl;
        struct pneigh_entry *pn = NULL;
        int bucket;

        state->flags |= NEIGH_SEQ_IS_PNEIGH;
        for (bucket = 0; bucket <= PNEIGH_HASHMASK; bucket++) {
                pn = tbl->phash_buckets[bucket];
                while (pn && !net_eq(pneigh_net(pn), net))
                        pn = pn->next;
                if (pn)
                        break;
        }
        state->bucket = bucket;

        return pn;
}

static struct pneigh_entry *pneigh_get_next(struct seq_file *seq,
                                            struct pneigh_entry *pn,
                                            loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        struct net *net = seq_file_net(seq);
        struct neigh_table *tbl = state->tbl;

        do {
                pn = pn->next;
        } while (pn && !net_eq(pneigh_net(pn), net));

        while (!pn) {
                if (++state->bucket > PNEIGH_HASHMASK)
                        break;
                pn = tbl->phash_buckets[state->bucket];
                while (pn && !net_eq(pneigh_net(pn), net))
                        pn = pn->next;
                if (pn)
                        break;
        }

        if (pn && pos)
                --(*pos);

        return pn;
}

static struct pneigh_entry *pneigh_get_idx(struct seq_file *seq, loff_t *pos)
{
        struct pneigh_entry *pn = pneigh_get_first(seq);

        if (pn) {
                --(*pos);
                while (*pos) {
                        pn = pneigh_get_next(seq, pn, pos);
                        if (!pn)
                                break;
                }
        }
        return *pos ? NULL : pn;
}

static void *neigh_get_idx_any(struct seq_file *seq, loff_t *pos)
{
        struct neigh_seq_state *state = seq->private;
        void *rc;
        loff_t idxpos = *pos;

        rc = neigh_get_idx(seq, &idxpos);
        if (!rc && !(state->flags & NEIGH_SEQ_NEIGH_ONLY))
                rc = pneigh_get_idx(seq, &idxpos);

        return rc;
}

void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl, unsigned int neigh_seq_flags)
        __acquires(tbl->lock)
        __acquires(rcu)
{
        struct neigh_seq_state *state = seq->private;

        state->tbl = tbl;
        state->bucket = -1;
        state->flags = (neigh_seq_flags & ~NEIGH_SEQ_IS_PNEIGH);

        rcu_read_lock();
        state->nht = rcu_dereference(tbl->nht);
        read_lock_bh(&tbl->lock);

        return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
}
EXPORT_SYMBOL(neigh_seq_start);

void *neigh_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct neigh_seq_state *state;
        void *rc;

        if (v == SEQ_START_TOKEN) {
                rc = neigh_get_first(seq);
                goto out;
        }

        state = seq->private;
        if (!(state->flags & NEIGH_SEQ_IS_PNEIGH)) {
                rc = neigh_get_next(seq, v, NULL);
                if (rc)
                        goto out;
                if (!(state->flags & NEIGH_SEQ_NEIGH_ONLY))
                        rc = pneigh_get_first(seq);
        } else {
                BUG_ON(state->flags & NEIGH_SEQ_NEIGH_ONLY);
                rc = pneigh_get_next(seq, v, NULL);
        }
out:
        ++(*pos);
        return rc;
}
EXPORT_SYMBOL(neigh_seq_next);

void neigh_seq_stop(struct seq_file *seq, void *v)
        __releases(tbl->lock)
        __releases(rcu)
{
        struct neigh_seq_state *state = seq->private;
        struct neigh_table *tbl = state->tbl;

        read_unlock_bh(&tbl->lock);
        rcu_read_unlock();
}
EXPORT_SYMBOL(neigh_seq_stop);

/* statistics via seq_file */

static void *neigh_stat_seq_start(struct seq_file *seq, loff_t *pos)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        int cpu;

        if (*pos == 0)
                return SEQ_START_TOKEN;

        for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu+1;
                return per_cpu_ptr(tbl->stats, cpu);
        }
        return NULL;
}

static void *neigh_stat_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        int cpu;

        for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu+1;
                return per_cpu_ptr(tbl->stats, cpu);
        }
        (*pos)++;
        return NULL;
}

static void neigh_stat_seq_stop(struct seq_file *seq, void *v)
{

}

static int neigh_stat_seq_show(struct seq_file *seq, void *v)
{
        struct neigh_table *tbl = pde_data(file_inode(seq->file));
        struct neigh_statistics *st = v;

        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, "entries  allocs   destroys hash_grows lookups  hits     res_failed rcv_probes_mcast rcv_probes_ucast periodic_gc_runs forced_gc_runs unresolved_discards table_fulls\n");
                return 0;
        }

        seq_printf(seq, "%08x %08lx %08lx %08lx   %08lx %08lx %08lx   "
                        "%08lx         %08lx         %08lx         "
                        "%08lx       %08lx            %08lx\n",
                   atomic_read(&tbl->entries),

                   st->allocs,
                   st->destroys,
                   st->hash_grows,

                   st->lookups,
                   st->hits,

                   st->res_failed,

                   st->rcv_probes_mcast,
                   st->rcv_probes_ucast,

                   st->periodic_gc_runs,
                   st->forced_gc_runs,
                   st->unres_discards,
                   st->table_fulls
                   );

        return 0;
}

static const struct seq_operations neigh_stat_seq_ops = {
        .start        = neigh_stat_seq_start,
        .next        = neigh_stat_seq_next,
        .stop        = neigh_stat_seq_stop,
        .show        = neigh_stat_seq_show,
};
#endif /* CONFIG_PROC_FS */

static void __neigh_notify(struct neighbour *n, int type, int flags,
                           u32 pid)
{
        struct sk_buff *skb;
        int err = -ENOBUFS;
        struct net *net;

        rcu_read_lock();
        net = dev_net_rcu(n->dev);
        skb = nlmsg_new(neigh_nlmsg_size(), GFP_ATOMIC);
        if (skb == NULL)
                goto errout;

        err = neigh_fill_info(skb, n, pid, 0, type, flags);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in neigh_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }
        rtnl_notify(skb, net, 0, RTNLGRP_NEIGH, NULL, GFP_ATOMIC);
        goto out;
errout:
        rtnl_set_sk_err(net, RTNLGRP_NEIGH, err);
out:
        rcu_read_unlock();
}

void neigh_app_ns(struct neighbour *n)
{
        __neigh_notify(n, RTM_GETNEIGH, NLM_F_REQUEST, 0);
}
EXPORT_SYMBOL(neigh_app_ns);

#ifdef CONFIG_SYSCTL
static int unres_qlen_max = INT_MAX / SKB_TRUESIZE(ETH_FRAME_LEN);

static int proc_unres_qlen(const struct ctl_table *ctl, int write,
                           void *buffer, size_t *lenp, loff_t *ppos)
{
        int size, ret;
        struct ctl_table tmp = *ctl;

        tmp.extra1 = SYSCTL_ZERO;
        tmp.extra2 = &unres_qlen_max;
        tmp.data = &size;

        size = *(int *)ctl->data / SKB_TRUESIZE(ETH_FRAME_LEN);
        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);

        if (write && !ret)
                *(int *)ctl->data = size * SKB_TRUESIZE(ETH_FRAME_LEN);
        return ret;
}

static void neigh_copy_dflt_parms(struct net *net, struct neigh_parms *p,
                                  int index)
{
        struct net_device *dev;
        int family = neigh_parms_family(p);

        rcu_read_lock();
        for_each_netdev_rcu(net, dev) {
                struct neigh_parms *dst_p =
                                neigh_get_dev_parms_rcu(dev, family);

                if (dst_p && !test_bit(index, dst_p->data_state))
                        dst_p->data[index] = p->data[index];
        }
        rcu_read_unlock();
}

static void neigh_proc_update(const struct ctl_table *ctl, int write)
{
        struct net_device *dev = ctl->extra1;
        struct neigh_parms *p = ctl->extra2;
        struct net *net = neigh_parms_net(p);
        int index = (int *) ctl->data - p->data;

        if (!write)
                return;

        set_bit(index, p->data_state);
        if (index == NEIGH_VAR_DELAY_PROBE_TIME)
                call_netevent_notifiers(NETEVENT_DELAY_PROBE_TIME_UPDATE, p);
        if (!dev) /* NULL dev means this is default value */
                neigh_copy_dflt_parms(net, p, index);
}

static int neigh_proc_dointvec_zero_intmax(const struct ctl_table *ctl, int write,
                                           void *buffer, size_t *lenp,
                                           loff_t *ppos)
{
        struct ctl_table tmp = *ctl;
        int ret;

        tmp.extra1 = SYSCTL_ZERO;
        tmp.extra2 = SYSCTL_INT_MAX;

        ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos);
        neigh_proc_update(ctl, write);
        return ret;
}

static int neigh_proc_dointvec_ms_jiffies_positive(const struct ctl_table *ctl, int write,
                                                   void *buffer, size_t *lenp, loff_t *ppos)
{
        struct ctl_table tmp = *ctl;
        int ret;

        int min = msecs_to_jiffies(1);

        tmp.extra1 = &min;
        tmp.extra2 = NULL;

        ret = proc_dointvec_ms_jiffies_minmax(&tmp, write, buffer, lenp, ppos);
        neigh_proc_update(ctl, write);
        return ret;
}

int neigh_proc_dointvec(const struct ctl_table *ctl, int write, void *buffer,
                        size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec);

int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write, void *buffer,
                                size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec_jiffies);

static int neigh_proc_dointvec_userhz_jiffies(const struct ctl_table *ctl, int write,
                                              void *buffer, size_t *lenp,
                                              loff_t *ppos)
{
        int ret = proc_dointvec_userhz_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}

int neigh_proc_dointvec_ms_jiffies(const struct ctl_table *ctl, int write,
                                   void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret = proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}
EXPORT_SYMBOL(neigh_proc_dointvec_ms_jiffies);

static int neigh_proc_dointvec_unres_qlen(const struct ctl_table *ctl, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        int ret = proc_unres_qlen(ctl, write, buffer, lenp, ppos);

        neigh_proc_update(ctl, write);
        return ret;
}

static int neigh_proc_base_reachable_time(const struct ctl_table *ctl, int write,
                                          void *buffer, size_t *lenp,
                                          loff_t *ppos)
{
        struct neigh_parms *p = ctl->extra2;
        int ret;

        if (strcmp(ctl->procname, "base_reachable_time") == 0)
                ret = neigh_proc_dointvec_jiffies(ctl, write, buffer, lenp, ppos);
        else if (strcmp(ctl->procname, "base_reachable_time_ms") == 0)
                ret = neigh_proc_dointvec_ms_jiffies(ctl, write, buffer, lenp, ppos);
        else
                ret = -1;

        if (write && ret == 0) {
                /* update reachable_time as well, otherwise, the change will
                 * only be effective after the next time neigh_periodic_work
                 * decides to recompute it
                 */
                p->reachable_time =
                        neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME));
        }
        return ret;
}

#define NEIGH_PARMS_DATA_OFFSET(index)        \
        (&((struct neigh_parms *) 0)->data[index])

#define NEIGH_SYSCTL_ENTRY(attr, data_attr, name, mval, proc) \
        [NEIGH_VAR_ ## attr] = { \
                .procname        = name, \
                .data                = NEIGH_PARMS_DATA_OFFSET(NEIGH_VAR_ ## data_attr), \
                .maxlen                = sizeof(int), \
                .mode                = mval, \
                .proc_handler        = proc, \
        }

#define NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_zero_intmax)

#define NEIGH_SYSCTL_JIFFIES_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_jiffies)

#define NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_userhz_jiffies)

#define NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, attr, name, 0644, neigh_proc_dointvec_ms_jiffies_positive)

#define NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(attr, data_attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_ms_jiffies)

#define NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(attr, data_attr, name) \
        NEIGH_SYSCTL_ENTRY(attr, data_attr, name, 0644, neigh_proc_dointvec_unres_qlen)

static struct neigh_sysctl_table {
        struct ctl_table_header *sysctl_header;
        struct ctl_table neigh_vars[NEIGH_VAR_MAX];
} neigh_sysctl_template __read_mostly = {
        .neigh_vars = {
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_PROBES, "mcast_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(UCAST_PROBES, "ucast_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(APP_PROBES, "app_solicit"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(MCAST_REPROBES, "mcast_resolicit"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(RETRANS_TIME, "retrans_time"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(BASE_REACHABLE_TIME, "base_reachable_time"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(DELAY_PROBE_TIME, "delay_first_probe_time"),
                NEIGH_SYSCTL_MS_JIFFIES_POSITIVE_ENTRY(INTERVAL_PROBE_TIME_MS,
                                                       "interval_probe_time_ms"),
                NEIGH_SYSCTL_JIFFIES_ENTRY(GC_STALETIME, "gc_stale_time"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(QUEUE_LEN_BYTES, "unres_qlen_bytes"),
                NEIGH_SYSCTL_ZERO_INTMAX_ENTRY(PROXY_QLEN, "proxy_qlen"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(ANYCAST_DELAY, "anycast_delay"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(PROXY_DELAY, "proxy_delay"),
                NEIGH_SYSCTL_USERHZ_JIFFIES_ENTRY(LOCKTIME, "locktime"),
                NEIGH_SYSCTL_UNRES_QLEN_REUSED_ENTRY(QUEUE_LEN, QUEUE_LEN_BYTES, "unres_qlen"),
                NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(RETRANS_TIME_MS, RETRANS_TIME, "retrans_time_ms"),
                NEIGH_SYSCTL_MS_JIFFIES_REUSED_ENTRY(BASE_REACHABLE_TIME_MS, BASE_REACHABLE_TIME, "base_reachable_time_ms"),
                [NEIGH_VAR_GC_INTERVAL] = {
                        .procname        = "gc_interval",
                        .maxlen                = sizeof(int),
                        .mode                = 0644,
                        .proc_handler        = proc_dointvec_jiffies,
                },
                [NEIGH_VAR_GC_THRESH1] = {
                        .procname        = "gc_thresh1",
                        .maxlen                = sizeof(int),
                        .mode                = 0644,
                        .extra1                = SYSCTL_ZERO,
                        .extra2                = SYSCTL_INT_MAX,
                        .proc_handler        = proc_dointvec_minmax,
                },
                [NEIGH_VAR_GC_THRESH2] = {
                        .procname        = "gc_thresh2",
                        .maxlen                = sizeof(int),
                        .mode                = 0644,
                        .extra1                = SYSCTL_ZERO,
                        .extra2                = SYSCTL_INT_MAX,
                        .proc_handler        = proc_dointvec_minmax,
                },
                [NEIGH_VAR_GC_THRESH3] = {
                        .procname        = "gc_thresh3",
                        .maxlen                = sizeof(int),
                        .mode                = 0644,
                        .extra1                = SYSCTL_ZERO,
                        .extra2                = SYSCTL_INT_MAX,
                        .proc_handler        = proc_dointvec_minmax,
                },
        },
};

int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p,
                          proc_handler *handler)
{
        int i;
        struct neigh_sysctl_table *t;
        const char *dev_name_source;
        char neigh_path[ sizeof("net//neigh/") + IFNAMSIZ + IFNAMSIZ ];
        char *p_name;
        size_t neigh_vars_size;

        t = kmemdup(&neigh_sysctl_template, sizeof(*t), GFP_KERNEL_ACCOUNT);
        if (!t)
                goto err;

        for (i = 0; i < NEIGH_VAR_GC_INTERVAL; i++) {
                t->neigh_vars[i].data += (long) p;
                t->neigh_vars[i].extra1 = dev;
                t->neigh_vars[i].extra2 = p;
        }

        neigh_vars_size = ARRAY_SIZE(t->neigh_vars);
        if (dev) {
                dev_name_source = dev->name;
                /* Terminate the table early */
                neigh_vars_size = NEIGH_VAR_BASE_REACHABLE_TIME_MS + 1;
        } else {
                struct neigh_table *tbl = p->tbl;
                dev_name_source = "default";
                t->neigh_vars[NEIGH_VAR_GC_INTERVAL].data = &tbl->gc_interval;
                t->neigh_vars[NEIGH_VAR_GC_THRESH1].data = &tbl->gc_thresh1;
                t->neigh_vars[NEIGH_VAR_GC_THRESH2].data = &tbl->gc_thresh2;
                t->neigh_vars[NEIGH_VAR_GC_THRESH3].data = &tbl->gc_thresh3;
        }

        if (handler) {
                /* RetransTime */
                t->neigh_vars[NEIGH_VAR_RETRANS_TIME].proc_handler = handler;
                /* ReachableTime */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler = handler;
                /* RetransTime (in milliseconds)*/
                t->neigh_vars[NEIGH_VAR_RETRANS_TIME_MS].proc_handler = handler;
                /* ReachableTime (in milliseconds) */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler = handler;
        } else {
                /* Those handlers will update p->reachable_time after
                 * base_reachable_time(_ms) is set to ensure the new timer starts being
                 * applied after the next neighbour update instead of waiting for
                 * neigh_periodic_work to update its value (can be multiple minutes)
                 * So any handler that replaces them should do this as well
                 */
                /* ReachableTime */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME].proc_handler =
                        neigh_proc_base_reachable_time;
                /* ReachableTime (in milliseconds) */
                t->neigh_vars[NEIGH_VAR_BASE_REACHABLE_TIME_MS].proc_handler =
                        neigh_proc_base_reachable_time;
        }

        switch (neigh_parms_family(p)) {
        case AF_INET:
              p_name = "ipv4";
              break;
        case AF_INET6:
              p_name = "ipv6";
              break;
        default:
              BUG();
        }

        snprintf(neigh_path, sizeof(neigh_path), "net/%s/neigh/%s",
                p_name, dev_name_source);
        t->sysctl_header = register_net_sysctl_sz(neigh_parms_net(p),
                                                  neigh_path, t->neigh_vars,
                                                  neigh_vars_size);
        if (!t->sysctl_header)
                goto free;

        p->sysctl_table = t;
        return 0;

free:
        kfree(t);
err:
        return -ENOBUFS;
}
EXPORT_SYMBOL(neigh_sysctl_register);

void neigh_sysctl_unregister(struct neigh_parms *p)
{
        if (p->sysctl_table) {
                struct neigh_sysctl_table *t = p->sysctl_table;
                p->sysctl_table = NULL;
                unregister_net_sysctl_table(t->sysctl_header);
                kfree(t);
        }
}
EXPORT_SYMBOL(neigh_sysctl_unregister);

#endif        /* CONFIG_SYSCTL */

static const struct rtnl_msg_handler neigh_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNEIGH, .doit = neigh_add},
        {.msgtype = RTM_DELNEIGH, .doit = neigh_delete},
        {.msgtype = RTM_GETNEIGH, .doit = neigh_get, .dumpit = neigh_dump_info,
         .flags = RTNL_FLAG_DUMP_UNLOCKED},
        {.msgtype = RTM_GETNEIGHTBL, .dumpit = neightbl_dump_info},
        {.msgtype = RTM_SETNEIGHTBL, .doit = neightbl_set},
};

static int __init neigh_init(void)
{
        rtnl_register_many(neigh_rtnl_msg_handlers);
        return 0;
}

subsys_initcall(neigh_init);













  442 

















































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ARM64_KVM_NESTED_H
#define __ARM64_KVM_NESTED_H

#include <linux/bitfield.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_pgtable.h>

static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu)
{
        return (!__is_defined(__KVM_NVHE_HYPERVISOR__) &&
                cpus_have_final_cap(ARM64_HAS_NESTED_VIRT) &&
                vcpu_has_feature(vcpu, KVM_ARM_VCPU_HAS_EL2));
}

/* Translation helpers from non-VHE EL2 to EL1 */
static inline u64 tcr_el2_ps_to_tcr_el1_ips(u64 tcr_el2)
{
        return (u64)FIELD_GET(TCR_EL2_PS_MASK, tcr_el2) << TCR_IPS_SHIFT;
}

static inline u64 translate_tcr_el2_to_tcr_el1(u64 tcr)
{
        return TCR_EPD1_MASK |                                /* disable TTBR1_EL1 */
               ((tcr & TCR_EL2_TBI) ? TCR_TBI0 : 0) |
               tcr_el2_ps_to_tcr_el1_ips(tcr) |
               (tcr & TCR_EL2_TG0_MASK) |
               (tcr & TCR_EL2_ORGN0_MASK) |
               (tcr & TCR_EL2_IRGN0_MASK) |
               (tcr & TCR_EL2_T0SZ_MASK);
}

static inline u64 translate_cptr_el2_to_cpacr_el1(u64 cptr_el2)
{
        u64 cpacr_el1 = CPACR_EL1_RES1;

        if (cptr_el2 & CPTR_EL2_TTA)
                cpacr_el1 |= CPACR_EL1_TTA;
        if (!(cptr_el2 & CPTR_EL2_TFP))
                cpacr_el1 |= CPACR_EL1_FPEN;
        if (!(cptr_el2 & CPTR_EL2_TZ))
                cpacr_el1 |= CPACR_EL1_ZEN;

        cpacr_el1 |= cptr_el2 & (CPTR_EL2_TCPAC | CPTR_EL2_TAM);

        return cpacr_el1;
}

static inline u64 translate_sctlr_el2_to_sctlr_el1(u64 val)
{
        /* Only preserve the minimal set of bits we support */
        val &= (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | SCTLR_ELx_SA |
                SCTLR_ELx_I | SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_EE);
        val |= SCTLR_EL1_RES1;

        return val;
}

static inline u64 translate_ttbr0_el2_to_ttbr0_el1(u64 ttbr0)
{
        /* Clear the ASID field */
        return ttbr0 & ~GENMASK_ULL(63, 48);
}

extern bool forward_smc_trap(struct kvm_vcpu *vcpu);
extern bool forward_debug_exception(struct kvm_vcpu *vcpu);
extern void kvm_init_nested(struct kvm *kvm);
extern int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu);
extern void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu);
extern struct kvm_s2_mmu *lookup_s2_mmu(struct kvm_vcpu *vcpu);

union tlbi_info;

extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
                                       const union tlbi_info *info,
                                       void (*)(struct kvm_s2_mmu *,
                                                const union tlbi_info *));
extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);

extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);

struct kvm_s2_trans {
        phys_addr_t output;
        unsigned long block_size;
        bool writable;
        bool readable;
        int level;
        u32 esr;
        u64 desc;
};

static inline phys_addr_t kvm_s2_trans_output(struct kvm_s2_trans *trans)
{
        return trans->output;
}

static inline unsigned long kvm_s2_trans_size(struct kvm_s2_trans *trans)
{
        return trans->block_size;
}

static inline u32 kvm_s2_trans_esr(struct kvm_s2_trans *trans)
{
        return trans->esr;
}

static inline bool kvm_s2_trans_readable(struct kvm_s2_trans *trans)
{
        return trans->readable;
}

static inline bool kvm_s2_trans_writable(struct kvm_s2_trans *trans)
{
        return trans->writable;
}

static inline bool kvm_s2_trans_executable(struct kvm_s2_trans *trans)
{
        return !(trans->desc & BIT(54));
}

extern int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa,
                              struct kvm_s2_trans *result);
extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
                                    struct kvm_s2_trans *trans);
extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
extern void kvm_nested_s2_wp(struct kvm *kvm);
extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
extern void kvm_nested_s2_flush(struct kvm *kvm);

unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val);

static inline bool kvm_supported_tlbi_s1e1_op(struct kvm_vcpu *vpcu, u32 instr)
{
        struct kvm *kvm = vpcu->kvm;
        u8 CRm = sys_reg_CRm(instr);

        if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
              sys_reg_Op1(instr) == TLBI_Op1_EL1))
                return false;

        if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
              (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
               kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
                return false;

        if (CRm == TLBI_CRm_nROS &&
            !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
                return false;

        if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
             CRm == TLBI_CRm_RNS) &&
            !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
                return false;

        return true;
}

static inline bool kvm_supported_tlbi_s1e2_op(struct kvm_vcpu *vpcu, u32 instr)
{
        struct kvm *kvm = vpcu->kvm;
        u8 CRm = sys_reg_CRm(instr);

        if (!(sys_reg_Op0(instr) == TLBI_Op0 &&
              sys_reg_Op1(instr) == TLBI_Op1_EL2))
                return false;

        if (!(sys_reg_CRn(instr) == TLBI_CRn_XS ||
              (sys_reg_CRn(instr) == TLBI_CRn_nXS &&
               kvm_has_feat(kvm, ID_AA64ISAR1_EL1, XS, IMP))))
                return false;

        if (CRm == TLBI_CRm_IPAIS || CRm == TLBI_CRm_IPAONS)
                return false;

        if (CRm == TLBI_CRm_nROS &&
            !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, OS))
                return false;

        if ((CRm == TLBI_CRm_RIS || CRm == TLBI_CRm_ROS ||
             CRm == TLBI_CRm_RNS) &&
            !kvm_has_feat(kvm, ID_AA64ISAR0_EL1, TLB, RANGE))
                return false;

        return true;
}

int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu);
u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val);

#ifdef CONFIG_ARM64_PTR_AUTH
bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr);
#else
static inline bool kvm_auth_eretax(struct kvm_vcpu *vcpu, u64 *elr)
{
        /* We really should never execute this... */
        WARN_ON_ONCE(1);
        *elr = 0xbad9acc0debadbad;
        return false;
}
#endif

#define KVM_NV_GUEST_MAP_SZ        (KVM_PGTABLE_PROT_SW1 | KVM_PGTABLE_PROT_SW0)

static inline u64 kvm_encode_nested_level(struct kvm_s2_trans *trans)
{
        return FIELD_PREP(KVM_NV_GUEST_MAP_SZ, trans->level);
}

/* Adjust alignment for the contiguous bit as per StageOA() */
#define contiguous_bit_shift(d, wi, l)                                        \
        ({                                                                \
                u8 shift = 0;                                                \
                                                                        \
                if ((d) & PTE_CONT) {                                        \
                        switch (BIT((wi)->pgshift)) {                        \
                        case SZ_4K:                                        \
                                shift = 4;                                \
                                break;                                        \
                        case SZ_16K:                                        \
                                shift = (l) == 2 ? 5 : 7;                \
                                break;                                        \
                        case SZ_64K:                                        \
                                shift = 5;                                \
                                break;                                        \
                        }                                                \
                }                                                        \
                                                                        \
                shift;                                                        \
        })

static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid)
{
        u64 base, tg, num, scale;
        int shift;

        tg        = FIELD_GET(GENMASK(47, 46), val);

        switch(tg) {
        case 1:
                shift = 12;
                break;
        case 2:
                shift = 14;
                break;
        case 3:
        default:                /* IMPDEF: handle tg==0 as 64k */
                shift = 16;
                break;
        }

        base        = (val & GENMASK(36, 0)) << shift;

        if (asid)
                *asid = FIELD_GET(TLBIR_ASID_MASK, val);

        scale        = FIELD_GET(GENMASK(45, 44), val);
        num        = FIELD_GET(GENMASK(43, 39), val);
        *range        = __TLBI_RANGE_PAGES(num, scale) << shift;

        return base;
}

static inline unsigned int ps_to_output_size(unsigned int ps)
{
        switch (ps) {
        case 0: return 32;
        case 1: return 36;
        case 2: return 40;
        case 3: return 42;
        case 4: return 44;
        case 5:
        default:
                return 48;
        }
}

enum trans_regime {
        TR_EL10,
        TR_EL20,
        TR_EL2,
};

struct s1_walk_info {
        u64                             baddr;
        enum trans_regime        regime;
        unsigned int                max_oa_bits;
        unsigned int                pgshift;
        unsigned int                txsz;
        int                              sl;
        bool                        as_el0;
        bool                             hpd;
        bool                        e0poe;
        bool                        poe;
        bool                        pan;
        bool                             be;
        bool                             s2;
};

struct s1_walk_result {
        union {
                struct {
                        u64        desc;
                        u64        pa;
                        s8        level;
                        u8        APTable;
                        bool        nG;
                        u16        asid;
                        bool        UXNTable;
                        bool        PXNTable;
                        bool        uwxn;
                        bool        uov;
                        bool        ur;
                        bool        uw;
                        bool        ux;
                        bool        pwxn;
                        bool        pov;
                        bool        pr;
                        bool        pw;
                        bool        px;
                };
                struct {
                        u8        fst;
                        bool        ptw;
                        bool        s2;
                };
        };
        bool        failed;
};

int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi,
                       struct s1_walk_result *wr, u64 va);

/* VNCR management */
int kvm_vcpu_allocate_vncr_tlb(struct kvm_vcpu *vcpu);
int kvm_handle_vncr_abort(struct kvm_vcpu *vcpu);
void kvm_handle_s1e2_tlbi(struct kvm_vcpu *vcpu, u32 inst, u64 val);

#define vncr_fixmap(c)                                                \
        ({                                                        \
                u32 __c = (c);                                        \
                BUG_ON(__c >= NR_CPUS);                                \
                (FIX_VNCR - __c);                                \
        })

#endif /* __ARM64_KVM_NESTED_H */



































































































































































  564 
  564 










   86 

   86 














  118 

   86 
   79 


  128 



























   39 

  129 



  165 
























  165 




  165 



  164 





























  318 






























  227 

  154 




  318 




  204 









  165 

  165 
  165 





  318 


  318 










  318 






   83 
  280 





   84 







  313 
    4 


  137 

   35 
  105 








   18 
  318 


  162 


    2 
  160 














  161 






  118 
  117 




  117 
    2 




    2 
    2 






    2 














  174 



  175 
  141 












































































































  434 





   88 



   88 

















































  421 




























































































































































































































































































































































































































  359 



  357 

    6 




  359 



  359 
















































































































































































































































































































































































  145 


























































   63 
   40 
   35 





   63 

   63 


  115 









   35 


  135 


  145 


























   22 





   22 



  145 
  145 

















  145 

















































































  175 




  145 
  131 
   96 




  130 

   20 




  147 






























   37 



  175 



   37 




  175 








  175 













  175 


  170 
   35 


  175 







  171 
   35 

  174 



































  251 










  160 



  175 


   79 























   92 

  252 

  252 
  313 












  313 




  313 
  313 








  312 




  313 



  313 

  313 

  313 

































   66 

   26 




  313 




























  318 













  318 
  318 

  318 

  318 
  318 
















    1 



    1 



    1 
    1 





    1 
    1 
    1 





























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































  349 
  349 

  349 









  349 



  349 

  349 


  349 



  349 

  349 
















  348 
  349 















  349 

  349 

  348 













  348 
  349 
  349 










  349 




  349 



  348 













  349 
  349 
  349 















  349 



  349 







  349 












  349 





  349 




  349 



  349 













  349 

























































    2 


























































































  397 
  276 
















































   17 


   17 

   17 













   16 










   14 


    3 


    3 





















































  164 

   17 



















   26 



  334 

   60 
   22 




   60 
   21 



































   37 
    2 
   37 
   14 







   37 


    2 





























   37 


    2 



   37 











   37 













   37 

   37 






















    2 





   37 


   13 


   37 

   37 
   15 
   37 

    2 
    2 
































































































































































































































































   37 






   37 
























   37 







    2 





   37 





















    2 














    2 

   37 

    2 


   37 













    1 



    1 






































































    7 

























    8 

    1 









    7 



























































































  401 

  277 



















































































































































































































































































































































































































































































































































































































































































































































  342 



















































































  398 











  167 















  344 
   39 


  342 





  344 

















  344 


























  401 





  398 


   16 











  176 



















  166 
   15 





   13 



  164 
















   90 

   79 












































































































  271 








  271 






  264 




   17 

    2 




  271 






  343 

  465 

  156 


















  164 




  156 
   19 













  163 




  155 

   13 



  129 
   50 






   50 




  164 






   90 


   79 




























  164 









  164 

  164 




  163 
















































































  158 
   45 




  188 



  188 








   86 



















   86 
  188 
  187 



  164 




    9 



  156 

  156 










    1 

    4 
    3 









    2 













    2 
    2 













   14 




    2 


   17 





   17 












    1 




   17 









































  266 

    4 

   19 


  273 
    8 















































































































































































































































































































  164 






















  504 



  505 





  551 




   39 




   39 



    8 

   37 




   18 


















   18 


















  546 








  551 



  548 



  541 
   49 

























  547 







  522 
  163 














  549 















  551 
























  576 







  576 








   56 



















  330 



    6 

  350 






  576 




























  408 




  377 


























  574 





  578 

  577 



  578 












  503 
  183 

  576 


   49 

  551 













  507 
  182 






  182 



  578 

























































   57 

   58 





   57 

   57 










































































































































































































































































































































































































































































































 1324 




 1325 













































































































































































































  172 






   10 




  379 
  346 
   42 



  375 
  347 
   42 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/mm/memory.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 */

/*
 * demand-loading started 01.12.91 - seems it is high on the list of
 * things wanted, and it should be easy to implement. - Linus
 */

/*
 * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
 * pages started 02.12.91, seems to work. - Linus.
 *
 * Tested sharing by executing about 30 /bin/sh: under the old kernel it
 * would have taken more than the 6M I have free, but it worked well as
 * far as I could see.
 *
 * Also corrected some "invalidate()"s - I wasn't doing enough of them.
 */

/*
 * Real VM (paging to/from disk) started 18.12.91. Much more work and
 * thought has to go into this. Oh, well..
 * 19.12.91  -  works, somewhat. Sometimes I get faults, don't know why.
 *                Found it. Everything seems to work now.
 * 20.12.91  -  Ok, making the swap-device changeable like the root.
 */

/*
 * 05.04.94  -  Multi-page memory management added for v1.1.
 *              Idea by Alex Bligh (alex@cconcepts.co.uk)
 *
 * 16.07.99  -  Support of BIGMEM added by Gerhard Wichert, Siemens AG
 *                (Gerhard.Wichert@pdb.siemens.de)
 *
 * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
 */

#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/sched/mm.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/hugetlb.h>
#include <linux/mman.h>
#include <linux/swap.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
#include <linux/memremap.h>
#include <linux/kmsan.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/export.h>
#include <linux/delayacct.h>
#include <linux/init.h>
#include <linux/pfn_t.h>
#include <linux/writeback.h>
#include <linux/memcontrol.h>
#include <linux/mmu_notifier.h>
#include <linux/swapops.h>
#include <linux/elf.h>
#include <linux/gfp.h>
#include <linux/migrate.h>
#include <linux/string.h>
#include <linux/memory-tiers.h>
#include <linux/debugfs.h>
#include <linux/userfaultfd_k.h>
#include <linux/dax.h>
#include <linux/oom.h>
#include <linux/numa.h>
#include <linux/perf_event.h>
#include <linux/ptrace.h>
#include <linux/vmalloc.h>
#include <linux/sched/sysctl.h>

#include <trace/events/kmem.h>

#include <asm/io.h>
#include <asm/mmu_context.h>
#include <asm/pgalloc.h>
#include <linux/uaccess.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>

#include "pgalloc-track.h"
#include "internal.h"
#include "swap.h"

#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
#endif

static vm_fault_t do_fault(struct vm_fault *vmf);
static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
static bool vmf_pte_changed(struct vm_fault *vmf);

/*
 * Return true if the original pte was a uffd-wp pte marker (so the pte was
 * wr-protected).
 */
static __always_inline bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
{
        if (!userfaultfd_wp(vmf->vma))
                return false;
        if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
                return false;

        return pte_marker_uffd_wp(vmf->orig_pte);
}

/*
 * Randomize the address space (stacks, mmaps, brk, etc.).
 *
 * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
 *   as ancient (libc5 based) binaries can segfault. )
 */
int randomize_va_space __read_mostly =
#ifdef CONFIG_COMPAT_BRK
                                        1;
#else
                                        2;
#endif

#ifndef arch_wants_old_prefaulted_pte
static inline bool arch_wants_old_prefaulted_pte(void)
{
        /*
         * Transitioning a PTE from 'old' to 'young' can be expensive on
         * some architectures, even if it's performed in hardware. By
         * default, "false" means prefaulted entries will be 'young'.
         */
        return false;
}
#endif

static int __init disable_randmaps(char *s)
{
        randomize_va_space = 0;
        return 1;
}
__setup("norandmaps", disable_randmaps);

unsigned long zero_pfn __read_mostly;
EXPORT_SYMBOL(zero_pfn);

unsigned long highest_memmap_pfn __read_mostly;

/*
 * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
 */
static int __init init_zero_pfn(void)
{
        zero_pfn = page_to_pfn(ZERO_PAGE(0));
        return 0;
}
early_initcall(init_zero_pfn);

void mm_trace_rss_stat(struct mm_struct *mm, int member)
{
        trace_rss_stat(mm, member);
}

/*
 * Note: this doesn't free the actual pages themselves. That
 * has been handled earlier when unmapping all the memory regions.
 */
static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
                           unsigned long addr)
{
        pgtable_t token = pmd_pgtable(*pmd);
        pmd_clear(pmd);
        pte_free_tlb(tlb, token, addr);
        mm_dec_nr_ptes(tlb->mm);
}

static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        pmd_t *pmd;
        unsigned long next;
        unsigned long start;

        start = addr;
        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (pmd_none_or_clear_bad(pmd))
                        continue;
                free_pte_range(tlb, pmd, addr);
        } while (pmd++, addr = next, addr != end);

        start &= PUD_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= PUD_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        pmd = pmd_offset(pud, start);
        pud_clear(pud);
        pmd_free_tlb(tlb, pmd, start);
        mm_dec_nr_pmds(tlb->mm);
}

static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        pud_t *pud;
        unsigned long next;
        unsigned long start;

        start = addr;
        pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_none_or_clear_bad(pud))
                        continue;
                free_pmd_range(tlb, pud, addr, next, floor, ceiling);
        } while (pud++, addr = next, addr != end);

        start &= P4D_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= P4D_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        pud = pud_offset(p4d, start);
        p4d_clear(p4d);
        pud_free_tlb(tlb, pud, start);
        mm_dec_nr_puds(tlb->mm);
}

static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd,
                                unsigned long addr, unsigned long end,
                                unsigned long floor, unsigned long ceiling)
{
        p4d_t *p4d;
        unsigned long next;
        unsigned long start;

        start = addr;
        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d))
                        continue;
                free_pud_range(tlb, p4d, addr, next, floor, ceiling);
        } while (p4d++, addr = next, addr != end);

        start &= PGDIR_MASK;
        if (start < floor)
                return;
        if (ceiling) {
                ceiling &= PGDIR_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                return;

        p4d = p4d_offset(pgd, start);
        pgd_clear(pgd);
        p4d_free_tlb(tlb, p4d, start);
}

/**
 * free_pgd_range - Unmap and free page tables in the range
 * @tlb: the mmu_gather containing pending TLB flush info
 * @addr: virtual address start
 * @end: virtual address end
 * @floor: lowest address boundary
 * @ceiling: highest address boundary
 *
 * This function tears down all user-level page tables in the
 * specified virtual address range [@addr..@end). It is part of
 * the memory unmap flow.
 */
void free_pgd_range(struct mmu_gather *tlb,
                        unsigned long addr, unsigned long end,
                        unsigned long floor, unsigned long ceiling)
{
        pgd_t *pgd;
        unsigned long next;

        /*
         * The next few lines have given us lots of grief...
         *
         * Why are we testing PMD* at this top level?  Because often
         * there will be no work to do at all, and we'd prefer not to
         * go all the way down to the bottom just to discover that.
         *
         * Why all these "- 1"s?  Because 0 represents both the bottom
         * of the address space and the top of it (using -1 for the
         * top wouldn't help much: the masks would do the wrong thing).
         * The rule is that addr 0 and floor 0 refer to the bottom of
         * the address space, but end 0 and ceiling 0 refer to the top
         * Comparisons need to use "end - 1" and "ceiling - 1" (though
         * that end 0 case should be mythical).
         *
         * Wherever addr is brought up or ceiling brought down, we must
         * be careful to reject "the opposite 0" before it confuses the
         * subsequent tests.  But what about where end is brought down
         * by PMD_SIZE below? no, end can't go down to 0 there.
         *
         * Whereas we round start (addr) and ceiling down, by different
         * masks at different levels, in order to test whether a table
         * now has no other vmas using it, so can be freed, we don't
         * bother to round floor or end up - the tests don't need that.
         */

        addr &= PMD_MASK;
        if (addr < floor) {
                addr += PMD_SIZE;
                if (!addr)
                        return;
        }
        if (ceiling) {
                ceiling &= PMD_MASK;
                if (!ceiling)
                        return;
        }
        if (end - 1 > ceiling - 1)
                end -= PMD_SIZE;
        if (addr > end - 1)
                return;
        /*
         * We add page table cache pages with PAGE_SIZE,
         * (see pte_free_tlb()), flush the tlb if we need
         */
        tlb_change_page_size(tlb, PAGE_SIZE);
        pgd = pgd_offset(tlb->mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
                free_p4d_range(tlb, pgd, addr, next, floor, ceiling);
        } while (pgd++, addr = next, addr != end);
}

void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
                   struct vm_area_struct *vma, unsigned long floor,
                   unsigned long ceiling, bool mm_wr_locked)
{
        struct unlink_vma_file_batch vb;

        tlb_free_vmas(tlb);

        do {
                unsigned long addr = vma->vm_start;
                struct vm_area_struct *next;

                /*
                 * Note: USER_PGTABLES_CEILING may be passed as ceiling and may
                 * be 0.  This will underflow and is okay.
                 */
                next = mas_find(mas, ceiling - 1);
                if (unlikely(xa_is_zero(next)))
                        next = NULL;

                /*
                 * Hide vma from rmap and truncate_pagecache before freeing
                 * pgtables
                 */
                if (mm_wr_locked)
                        vma_start_write(vma);
                unlink_anon_vmas(vma);

                if (is_vm_hugetlb_page(vma)) {
                        unlink_file_vma(vma);
                        hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
                                floor, next ? next->vm_start : ceiling);
                } else {
                        unlink_file_vma_batch_init(&vb);
                        unlink_file_vma_batch_add(&vb, vma);

                        /*
                         * Optimization: gather nearby vmas into one call down
                         */
                        while (next && next->vm_start <= vma->vm_end + PMD_SIZE
                               && !is_vm_hugetlb_page(next)) {
                                vma = next;
                                next = mas_find(mas, ceiling - 1);
                                if (unlikely(xa_is_zero(next)))
                                        next = NULL;
                                if (mm_wr_locked)
                                        vma_start_write(vma);
                                unlink_anon_vmas(vma);
                                unlink_file_vma_batch_add(&vb, vma);
                        }
                        unlink_file_vma_batch_final(&vb);
                        free_pgd_range(tlb, addr, vma->vm_end,
                                floor, next ? next->vm_start : ceiling);
                }
                vma = next;
        } while (vma);
}

void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte)
{
        spinlock_t *ptl = pmd_lock(mm, pmd);

        if (likely(pmd_none(*pmd))) {        /* Has another populated it ? */
                mm_inc_nr_ptes(mm);
                /*
                 * Ensure all pte setup (eg. pte page lock and page clearing) are
                 * visible before the pte is made visible to other CPUs by being
                 * put into page tables.
                 *
                 * The other side of the story is the pointer chasing in the page
                 * table walking code (when walking the page table without locking;
                 * ie. most of the time). Fortunately, these data accesses consist
                 * of a chain of data-dependent loads, meaning most CPUs (alpha
                 * being the notable exception) will already guarantee loads are
                 * seen in-order. See the alpha page table accessors for the
                 * smp_rmb() barriers in page table walking code.
                 */
                smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
                pmd_populate(mm, pmd, *pte);
                *pte = NULL;
        }
        spin_unlock(ptl);
}

int __pte_alloc(struct mm_struct *mm, pmd_t *pmd)
{
        pgtable_t new = pte_alloc_one(mm);
        if (!new)
                return -ENOMEM;

        pmd_install(mm, pmd, &new);
        if (new)
                pte_free(mm, new);
        return 0;
}

int __pte_alloc_kernel(pmd_t *pmd)
{
        pte_t *new = pte_alloc_one_kernel(&init_mm);
        if (!new)
                return -ENOMEM;

        spin_lock(&init_mm.page_table_lock);
        if (likely(pmd_none(*pmd))) {        /* Has another populated it ? */
                smp_wmb(); /* See comment in pmd_install() */
                pmd_populate_kernel(&init_mm, pmd, new);
                new = NULL;
        }
        spin_unlock(&init_mm.page_table_lock);
        if (new)
                pte_free_kernel(&init_mm, new);
        return 0;
}

static inline void init_rss_vec(int *rss)
{
        memset(rss, 0, sizeof(int) * NR_MM_COUNTERS);
}

static inline void add_mm_rss_vec(struct mm_struct *mm, int *rss)
{
        int i;

        for (i = 0; i < NR_MM_COUNTERS; i++)
                if (rss[i])
                        add_mm_counter(mm, i, rss[i]);
}

/*
 * This function is called to print an error when a bad pte
 * is found. For example, we might have a PFN-mapped pte in
 * a region that doesn't allow it.
 *
 * The calling function must still handle the error.
 */
static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
                          pte_t pte, struct page *page)
{
        pgd_t *pgd = pgd_offset(vma->vm_mm, addr);
        p4d_t *p4d = p4d_offset(pgd, addr);
        pud_t *pud = pud_offset(p4d, addr);
        pmd_t *pmd = pmd_offset(pud, addr);
        struct address_space *mapping;
        pgoff_t index;
        static unsigned long resume;
        static unsigned long nr_shown;
        static unsigned long nr_unshown;

        /*
         * Allow a burst of 60 reports, then keep quiet for that minute;
         * or allow a steady drip of one report per second.
         */
        if (nr_shown == 60) {
                if (time_before(jiffies, resume)) {
                        nr_unshown++;
                        return;
                }
                if (nr_unshown) {
                        pr_alert("BUG: Bad page map: %lu messages suppressed\n",
                                 nr_unshown);
                        nr_unshown = 0;
                }
                nr_shown = 0;
        }
        if (nr_shown++ == 0)
                resume = jiffies + 60 * HZ;

        mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
        index = linear_page_index(vma, addr);

        pr_alert("BUG: Bad page map in process %s  pte:%08llx pmd:%08llx\n",
                 current->comm,
                 (long long)pte_val(pte), (long long)pmd_val(*pmd));
        if (page)
                dump_page(page, "bad pte");
        pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
                 (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
        pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n",
                 vma->vm_file,
                 vma->vm_ops ? vma->vm_ops->fault : NULL,
                 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
                 vma->vm_file ? vma->vm_file->f_op->mmap_prepare : NULL,
                 mapping ? mapping->a_ops->read_folio : NULL);
        dump_stack();
        add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}

/*
 * vm_normal_page -- This function gets the "struct page" associated with a pte.
 *
 * "Special" mappings do not wish to be associated with a "struct page" (either
 * it doesn't exist, or it exists but they don't want to touch it). In this
 * case, NULL is returned here. "Normal" mappings do have a struct page.
 *
 * There are 2 broad cases. Firstly, an architecture may define a pte_special()
 * pte bit, in which case this function is trivial. Secondly, an architecture
 * may not have a spare pte bit, which requires a more complicated scheme,
 * described below.
 *
 * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
 * special mapping (even if there are underlying and valid "struct pages").
 * COWed pages of a VM_PFNMAP are always normal.
 *
 * The way we recognize COWed pages within VM_PFNMAP mappings is through the
 * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
 * set, and the vm_pgoff will point to the first PFN mapped: thus every special
 * mapping will always honor the rule
 *
 *        pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
 *
 * And for normal mappings this is false.
 *
 * This restricts such mappings to be a linear translation from virtual address
 * to pfn. To get around this restriction, we allow arbitrary mappings so long
 * as the vma is not a COW mapping; in that case, we know that all ptes are
 * special (because none can have been COWed).
 *
 *
 * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
 *
 * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
 * page" backing, however the difference is that _all_ pages with a struct
 * page (that is, those where pfn_valid is true) are refcounted and considered
 * normal pages by the VM. The only exception are zeropages, which are
 * *never* refcounted.
 *
 * The disadvantage is that pages are refcounted (which can be slower and
 * simply not an option for some PFNMAP users). The advantage is that we
 * don't have to follow the strict linearity rule of PFNMAP mappings in
 * order to support COWable mappings.
 *
 */
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
                            pte_t pte)
{
        unsigned long pfn = pte_pfn(pte);

        if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
                if (likely(!pte_special(pte)))
                        goto check_pfn;
                if (vma->vm_ops && vma->vm_ops->find_special_page)
                        return vma->vm_ops->find_special_page(vma, addr);
                if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
                        return NULL;
                if (is_zero_pfn(pfn))
                        return NULL;
                if (pte_devmap(pte))
                /*
                 * NOTE: New users of ZONE_DEVICE will not set pte_devmap()
                 * and will have refcounts incremented on their struct pages
                 * when they are inserted into PTEs, thus they are safe to
                 * return here. Legacy ZONE_DEVICE pages that set pte_devmap()
                 * do not have refcounts. Example of legacy ZONE_DEVICE is
                 * MEMORY_DEVICE_FS_DAX type in pmem or virtio_fs drivers.
                 */
                        return NULL;

                print_bad_pte(vma, addr, pte, NULL);
                return NULL;
        }

        /* !CONFIG_ARCH_HAS_PTE_SPECIAL case follows: */

        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
                if (vma->vm_flags & VM_MIXEDMAP) {
                        if (!pfn_valid(pfn))
                                return NULL;
                        if (is_zero_pfn(pfn))
                                return NULL;
                        goto out;
                } else {
                        unsigned long off;
                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
                        if (pfn == vma->vm_pgoff + off)
                                return NULL;
                        if (!is_cow_mapping(vma->vm_flags))
                                return NULL;
                }
        }

        if (is_zero_pfn(pfn))
                return NULL;

check_pfn:
        if (unlikely(pfn > highest_memmap_pfn)) {
                print_bad_pte(vma, addr, pte, NULL);
                return NULL;
        }

        /*
         * NOTE! We still have PageReserved() pages in the page tables.
         * eg. VDSO mappings can cause them to exist.
         */
out:
        VM_WARN_ON_ONCE(is_zero_pfn(pfn));
        return pfn_to_page(pfn);
}

struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
                            pte_t pte)
{
        struct page *page = vm_normal_page(vma, addr, pte);

        if (page)
                return page_folio(page);
        return NULL;
}

#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
                                pmd_t pmd)
{
        unsigned long pfn = pmd_pfn(pmd);

        /* Currently it's only used for huge pfnmaps */
        if (unlikely(pmd_special(pmd)))
                return NULL;

        if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
                if (vma->vm_flags & VM_MIXEDMAP) {
                        if (!pfn_valid(pfn))
                                return NULL;
                        goto out;
                } else {
                        unsigned long off;
                        off = (addr - vma->vm_start) >> PAGE_SHIFT;
                        if (pfn == vma->vm_pgoff + off)
                                return NULL;
                        if (!is_cow_mapping(vma->vm_flags))
                                return NULL;
                }
        }

        if (pmd_devmap(pmd))
                return NULL;
        if (is_huge_zero_pmd(pmd))
                return NULL;
        if (unlikely(pfn > highest_memmap_pfn))
                return NULL;

        /*
         * NOTE! We still have PageReserved() pages in the page tables.
         * eg. VDSO mappings can cause them to exist.
         */
out:
        return pfn_to_page(pfn);
}

struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
                                  unsigned long addr, pmd_t pmd)
{
        struct page *page = vm_normal_page_pmd(vma, addr, pmd);

        if (page)
                return page_folio(page);
        return NULL;
}
#endif

/**
 * restore_exclusive_pte - Restore a device-exclusive entry
 * @vma: VMA covering @address
 * @folio: the mapped folio
 * @page: the mapped folio page
 * @address: the virtual address
 * @ptep: pte pointer into the locked page table mapping the folio page
 * @orig_pte: pte value at @ptep
 *
 * Restore a device-exclusive non-swap entry to an ordinary present pte.
 *
 * The folio and the page table must be locked, and MMU notifiers must have
 * been called to invalidate any (exclusive) device mappings.
 *
 * Locking the folio makes sure that anybody who just converted the pte to
 * a device-exclusive entry can map it into the device to make forward
 * progress without others converting it back until the folio was unlocked.
 *
 * If the folio lock ever becomes an issue, we can stop relying on the folio
 * lock; it might make some scenarios with heavy thrashing less likely to
 * make forward progress, but these scenarios might not be valid use cases.
 *
 * Note that the folio lock does not protect against all cases of concurrent
 * page table modifications (e.g., MADV_DONTNEED, mprotect), so device drivers
 * must use MMU notifiers to sync against any concurrent changes.
 */
static void restore_exclusive_pte(struct vm_area_struct *vma,
                struct folio *folio, struct page *page, unsigned long address,
                pte_t *ptep, pte_t orig_pte)
{
        pte_t pte;

        VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);

        pte = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
        if (pte_swp_soft_dirty(orig_pte))
                pte = pte_mksoft_dirty(pte);

        if (pte_swp_uffd_wp(orig_pte))
                pte = pte_mkuffd_wp(pte);

        if ((vma->vm_flags & VM_WRITE) &&
            can_change_pte_writable(vma, address, pte)) {
                if (folio_test_dirty(folio))
                        pte = pte_mkdirty(pte);
                pte = pte_mkwrite(pte, vma);
        }
        set_pte_at(vma->vm_mm, address, ptep, pte);

        /*
         * No need to invalidate - it was non-present before. However
         * secondary CPUs may have mappings that need invalidating.
         */
        update_mmu_cache(vma, address, ptep);
}

/*
 * Tries to restore an exclusive pte if the page lock can be acquired without
 * sleeping.
 */
static int try_restore_exclusive_pte(struct vm_area_struct *vma,
                unsigned long addr, pte_t *ptep, pte_t orig_pte)
{
        struct page *page = pfn_swap_entry_to_page(pte_to_swp_entry(orig_pte));
        struct folio *folio = page_folio(page);

        if (folio_trylock(folio)) {
                restore_exclusive_pte(vma, folio, page, addr, ptep, orig_pte);
                folio_unlock(folio);
                return 0;
        }

        return -EBUSY;
}

/*
 * copy one vm_area from one task to the other. Assumes the page tables
 * already present in the new task to be cleared in the whole range
 * covered by this vma.
 */

static unsigned long
copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
                pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, unsigned long addr, int *rss)
{
        unsigned long vm_flags = dst_vma->vm_flags;
        pte_t orig_pte = ptep_get(src_pte);
        pte_t pte = orig_pte;
        struct folio *folio;
        struct page *page;
        swp_entry_t entry = pte_to_swp_entry(orig_pte);

        if (likely(!non_swap_entry(entry))) {
                if (swap_duplicate(entry) < 0)
                        return -EIO;

                /* make sure dst_mm is on swapoff's mmlist. */
                if (unlikely(list_empty(&dst_mm->mmlist))) {
                        spin_lock(&mmlist_lock);
                        if (list_empty(&dst_mm->mmlist))
                                list_add(&dst_mm->mmlist,
                                                &src_mm->mmlist);
                        spin_unlock(&mmlist_lock);
                }
                /* Mark the swap entry as shared. */
                if (pte_swp_exclusive(orig_pte)) {
                        pte = pte_swp_clear_exclusive(orig_pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
                rss[MM_SWAPENTS]++;
        } else if (is_migration_entry(entry)) {
                folio = pfn_swap_entry_folio(entry);

                rss[mm_counter(folio)]++;

                if (!is_readable_migration_entry(entry) &&
                                is_cow_mapping(vm_flags)) {
                        /*
                         * COW mappings require pages in both parent and child
                         * to be set to read. A previously exclusive entry is
                         * now shared.
                         */
                        entry = make_readable_migration_entry(
                                                        swp_offset(entry));
                        pte = swp_entry_to_pte(entry);
                        if (pte_swp_soft_dirty(orig_pte))
                                pte = pte_swp_mksoft_dirty(pte);
                        if (pte_swp_uffd_wp(orig_pte))
                                pte = pte_swp_mkuffd_wp(pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
        } else if (is_device_private_entry(entry)) {
                page = pfn_swap_entry_to_page(entry);
                folio = page_folio(page);

                /*
                 * Update rss count even for unaddressable pages, as
                 * they should treated just like normal pages in this
                 * respect.
                 *
                 * We will likely want to have some new rss counters
                 * for unaddressable pages, at some point. But for now
                 * keep things as they are.
                 */
                folio_get(folio);
                rss[mm_counter(folio)]++;
                /* Cannot fail as these pages cannot get pinned. */
                folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma);

                /*
                 * We do not preserve soft-dirty information, because so
                 * far, checkpoint/restore is the only feature that
                 * requires that. And checkpoint/restore does not work
                 * when a device driver is involved (you cannot easily
                 * save and restore device driver state).
                 */
                if (is_writable_device_private_entry(entry) &&
                    is_cow_mapping(vm_flags)) {
                        entry = make_readable_device_private_entry(
                                                        swp_offset(entry));
                        pte = swp_entry_to_pte(entry);
                        if (pte_swp_uffd_wp(orig_pte))
                                pte = pte_swp_mkuffd_wp(pte);
                        set_pte_at(src_mm, addr, src_pte, pte);
                }
        } else if (is_device_exclusive_entry(entry)) {
                /*
                 * Make device exclusive entries present by restoring the
                 * original entry then copying as for a present pte. Device
                 * exclusive entries currently only support private writable
                 * (ie. COW) mappings.
                 */
                VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
                if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte))
                        return -EBUSY;
                return -ENOENT;
        } else if (is_pte_marker_entry(entry)) {
                pte_marker marker = copy_pte_marker(entry, dst_vma);

                if (marker)
                        set_pte_at(dst_mm, addr, dst_pte,
                                   make_pte_marker(marker));
                return 0;
        }
        if (!userfaultfd_wp(dst_vma))
                pte = pte_swp_clear_uffd_wp(pte);
        set_pte_at(dst_mm, addr, dst_pte, pte);
        return 0;
}

/*
 * Copy a present and normal page.
 *
 * NOTE! The usual case is that this isn't required;
 * instead, the caller can just increase the page refcount
 * and re-use the pte the traditional way.
 *
 * And if we need a pre-allocated page but don't yet have
 * one, return a negative error to let the preallocation
 * code know so that it can do so outside the page table
 * lock.
 */
static inline int
copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
                  pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss,
                  struct folio **prealloc, struct page *page)
{
        struct folio *new_folio;
        pte_t pte;

        new_folio = *prealloc;
        if (!new_folio)
                return -EAGAIN;

        /*
         * We have a prealloc page, all good!  Take it
         * over and copy the page & arm it.
         */

        if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma))
                return -EHWPOISON;

        *prealloc = NULL;
        __folio_mark_uptodate(new_folio);
        folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(new_folio, dst_vma);
        rss[MM_ANONPAGES]++;

        /* All done, just insert the new page copy in the child */
        pte = folio_mk_pte(new_folio, dst_vma->vm_page_prot);
        pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma);
        if (userfaultfd_pte_wp(dst_vma, ptep_get(src_pte)))
                /* Uffd-wp needs to be delivered to dest pte as well */
                pte = pte_mkuffd_wp(pte);
        set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte);
        return 0;
}

static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
                struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte,
                pte_t pte, unsigned long addr, int nr)
{
        struct mm_struct *src_mm = src_vma->vm_mm;

        /* If it's a COW mapping, write protect it both processes. */
        if (is_cow_mapping(src_vma->vm_flags) && pte_write(pte)) {
                wrprotect_ptes(src_mm, addr, src_pte, nr);
                pte = pte_wrprotect(pte);
        }

        /* If it's a shared mapping, mark it clean in the child. */
        if (src_vma->vm_flags & VM_SHARED)
                pte = pte_mkclean(pte);
        pte = pte_mkold(pte);

        if (!userfaultfd_wp(dst_vma))
                pte = pte_clear_uffd_wp(pte);

        set_ptes(dst_vma->vm_mm, addr, dst_pte, pte, nr);
}

/*
 * Copy one present PTE, trying to batch-process subsequent PTEs that map
 * consecutive pages of the same folio by copying them as well.
 *
 * Returns -EAGAIN if one preallocated page is required to copy the next PTE.
 * Otherwise, returns the number of copied PTEs (at least 1).
 */
static inline int
copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
                 pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr,
                 int max_nr, int *rss, struct folio **prealloc)
{
        struct page *page;
        struct folio *folio;
        bool any_writable;
        fpb_t flags = 0;
        int err, nr;

        page = vm_normal_page(src_vma, addr, pte);
        if (unlikely(!page))
                goto copy_pte;

        folio = page_folio(page);

        /*
         * If we likely have to copy, just don't bother with batching. Make
         * sure that the common "small folio" case is as fast as possible
         * by keeping the batching logic separate.
         */
        if (unlikely(!*prealloc && folio_test_large(folio) && max_nr != 1)) {
                if (src_vma->vm_flags & VM_SHARED)
                        flags |= FPB_IGNORE_DIRTY;
                if (!vma_soft_dirty_enabled(src_vma))
                        flags |= FPB_IGNORE_SOFT_DIRTY;

                nr = folio_pte_batch(folio, addr, src_pte, pte, max_nr, flags,
                                     &any_writable, NULL, NULL);
                folio_ref_add(folio, nr);
                if (folio_test_anon(folio)) {
                        if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page,
                                                                  nr, dst_vma, src_vma))) {
                                folio_ref_sub(folio, nr);
                                return -EAGAIN;
                        }
                        rss[MM_ANONPAGES] += nr;
                        VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
                } else {
                        folio_dup_file_rmap_ptes(folio, page, nr, dst_vma);
                        rss[mm_counter_file(folio)] += nr;
                }
                if (any_writable)
                        pte = pte_mkwrite(pte, src_vma);
                __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte,
                                    addr, nr);
                return nr;
        }

        folio_get(folio);
        if (folio_test_anon(folio)) {
                /*
                 * If this page may have been pinned by the parent process,
                 * copy the page immediately for the child so that we'll always
                 * guarantee the pinned page won't be randomly replaced in the
                 * future.
                 */
                if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) {
                        /* Page may be pinned, we have to copy. */
                        folio_put(folio);
                        err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
                                                addr, rss, prealloc, page);
                        return err ? err : 1;
                }
                rss[MM_ANONPAGES]++;
                VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio);
        } else {
                folio_dup_file_rmap_pte(folio, page, dst_vma);
                rss[mm_counter_file(folio)]++;
        }

copy_pte:
        __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, addr, 1);
        return 1;
}

static inline struct folio *folio_prealloc(struct mm_struct *src_mm,
                struct vm_area_struct *vma, unsigned long addr, bool need_zero)
{
        struct folio *new_folio;

        if (need_zero)
                new_folio = vma_alloc_zeroed_movable_folio(vma, addr);
        else
                new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr);

        if (!new_folio)
                return NULL;

        if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
                folio_put(new_folio);
                return NULL;
        }
        folio_throttle_swaprate(new_folio, GFP_KERNEL);

        return new_folio;
}

static int
copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pte_t *orig_src_pte, *orig_dst_pte;
        pte_t *src_pte, *dst_pte;
        pmd_t dummy_pmdval;
        pte_t ptent;
        spinlock_t *src_ptl, *dst_ptl;
        int progress, max_nr, ret = 0;
        int rss[NR_MM_COUNTERS];
        swp_entry_t entry = (swp_entry_t){0};
        struct folio *prealloc = NULL;
        int nr;

again:
        progress = 0;
        init_rss_vec(rss);

        /*
         * copy_pmd_range()'s prior pmd_none_or_clear_bad(src_pmd), and the
         * error handling here, assume that exclusive mmap_lock on dst and src
         * protects anon from unexpected THP transitions; with shmem and file
         * protected by mmap_lock-less collapse skipping areas with anon_vma
         * (whereas vma_needs_copy() skips areas without anon_vma).  A rework
         * can remove such assumptions later, but this is good enough for now.
         */
        dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
        if (!dst_pte) {
                ret = -ENOMEM;
                goto out;
        }

        /*
         * We already hold the exclusive mmap_lock, the copy_pte_range() and
         * retract_page_tables() are using vma->anon_vma to be exclusive, so
         * the PTE page is stable, and there is no need to get pmdval and do
         * pmd_same() check.
         */
        src_pte = pte_offset_map_rw_nolock(src_mm, src_pmd, addr, &dummy_pmdval,
                                           &src_ptl);
        if (!src_pte) {
                pte_unmap_unlock(dst_pte, dst_ptl);
                /* ret == 0 */
                goto out;
        }
        spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
        orig_src_pte = src_pte;
        orig_dst_pte = dst_pte;
        arch_enter_lazy_mmu_mode();

        do {
                nr = 1;

                /*
                 * We are holding two locks at this point - either of them
                 * could generate latencies in another task on another CPU.
                 */
                if (progress >= 32) {
                        progress = 0;
                        if (need_resched() ||
                            spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
                                break;
                }
                ptent = ptep_get(src_pte);
                if (pte_none(ptent)) {
                        progress++;
                        continue;
                }
                if (unlikely(!pte_present(ptent))) {
                        ret = copy_nonpresent_pte(dst_mm, src_mm,
                                                  dst_pte, src_pte,
                                                  dst_vma, src_vma,
                                                  addr, rss);
                        if (ret == -EIO) {
                                entry = pte_to_swp_entry(ptep_get(src_pte));
                                break;
                        } else if (ret == -EBUSY) {
                                break;
                        } else if (!ret) {
                                progress += 8;
                                continue;
                        }
                        ptent = ptep_get(src_pte);
                        VM_WARN_ON_ONCE(!pte_present(ptent));

                        /*
                         * Device exclusive entry restored, continue by copying
                         * the now present pte.
                         */
                        WARN_ON_ONCE(ret != -ENOENT);
                }
                /* copy_present_ptes() will clear `*prealloc' if consumed */
                max_nr = (end - addr) / PAGE_SIZE;
                ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte,
                                        ptent, addr, max_nr, rss, &prealloc);
                /*
                 * If we need a pre-allocated page for this pte, drop the
                 * locks, allocate, and try again.
                 * If copy failed due to hwpoison in source page, break out.
                 */
                if (unlikely(ret == -EAGAIN || ret == -EHWPOISON))
                        break;
                if (unlikely(prealloc)) {
                        /*
                         * pre-alloc page cannot be reused by next time so as
                         * to strictly follow mempolicy (e.g., alloc_page_vma()
                         * will allocate page according to address).  This
                         * could only happen if one pinned pte changed.
                         */
                        folio_put(prealloc);
                        prealloc = NULL;
                }
                nr = ret;
                progress += 8 * nr;
        } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr,
                 addr != end);

        arch_leave_lazy_mmu_mode();
        pte_unmap_unlock(orig_src_pte, src_ptl);
        add_mm_rss_vec(dst_mm, rss);
        pte_unmap_unlock(orig_dst_pte, dst_ptl);
        cond_resched();

        if (ret == -EIO) {
                VM_WARN_ON_ONCE(!entry.val);
                if (add_swap_count_continuation(entry, GFP_KERNEL) < 0) {
                        ret = -ENOMEM;
                        goto out;
                }
                entry.val = 0;
        } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) {
                goto out;
        } else if (ret ==  -EAGAIN) {
                prealloc = folio_prealloc(src_mm, src_vma, addr, false);
                if (!prealloc)
                        return -ENOMEM;
        } else if (ret < 0) {
                VM_WARN_ON_ONCE(1);
        }

        /* We've captured and resolved the error. Reset, try again. */
        ret = 0;

        if (addr != end)
                goto again;
out:
        if (unlikely(prealloc))
                folio_put(prealloc);
        return ret;
}

static inline int
copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pmd_t *src_pmd, *dst_pmd;
        unsigned long next;

        dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
        if (!dst_pmd)
                return -ENOMEM;
        src_pmd = pmd_offset(src_pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (is_swap_pmd(*src_pmd) || pmd_trans_huge(*src_pmd)
                        || pmd_devmap(*src_pmd)) {
                        int err;
                        VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
                        err = copy_huge_pmd(dst_mm, src_mm, dst_pmd, src_pmd,
                                            addr, dst_vma, src_vma);
                        if (err == -ENOMEM)
                                return -ENOMEM;
                        if (!err)
                                continue;
                        /* fall through */
                }
                if (pmd_none_or_clear_bad(src_pmd))
                        continue;
                if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_pmd++, src_pmd++, addr = next, addr != end);
        return 0;
}

static inline int
copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               p4d_t *dst_p4d, p4d_t *src_p4d, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        pud_t *src_pud, *dst_pud;
        unsigned long next;

        dst_pud = pud_alloc(dst_mm, dst_p4d, addr);
        if (!dst_pud)
                return -ENOMEM;
        src_pud = pud_offset(src_p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_trans_huge(*src_pud) || pud_devmap(*src_pud)) {
                        int err;

                        VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
                        err = copy_huge_pud(dst_mm, src_mm,
                                            dst_pud, src_pud, addr, src_vma);
                        if (err == -ENOMEM)
                                return -ENOMEM;
                        if (!err)
                                continue;
                        /* fall through */
                }
                if (pud_none_or_clear_bad(src_pud))
                        continue;
                if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_pud++, src_pud++, addr = next, addr != end);
        return 0;
}

static inline int
copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
               pgd_t *dst_pgd, pgd_t *src_pgd, unsigned long addr,
               unsigned long end)
{
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        p4d_t *src_p4d, *dst_p4d;
        unsigned long next;

        dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr);
        if (!dst_p4d)
                return -ENOMEM;
        src_p4d = p4d_offset(src_pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(src_p4d))
                        continue;
                if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
                                   addr, next))
                        return -ENOMEM;
        } while (dst_p4d++, src_p4d++, addr = next, addr != end);
        return 0;
}

/*
 * Return true if the vma needs to copy the pgtable during this fork().  Return
 * false when we can speed up fork() by allowing lazy page faults later until
 * when the child accesses the memory range.
 */
static bool
vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
        /*
         * Always copy pgtables when dst_vma has uffd-wp enabled even if it's
         * file-backed (e.g. shmem). Because when uffd-wp is enabled, pgtable
         * contains uffd-wp protection information, that's something we can't
         * retrieve from page cache, and skip copying will lose those info.
         */
        if (userfaultfd_wp(dst_vma))
                return true;

        if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
                return true;

        if (src_vma->anon_vma)
                return true;

        /*
         * Don't copy ptes where a page fault will fill them correctly.  Fork
         * becomes much lighter when there are big shared or private readonly
         * mappings. The tradeoff is that copy_page_range is more efficient
         * than faulting.
         */
        return false;
}

int
copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
{
        pgd_t *src_pgd, *dst_pgd;
        unsigned long addr = src_vma->vm_start;
        unsigned long end = src_vma->vm_end;
        struct mm_struct *dst_mm = dst_vma->vm_mm;
        struct mm_struct *src_mm = src_vma->vm_mm;
        struct mmu_notifier_range range;
        unsigned long next;
        bool is_cow;
        int ret;

        if (!vma_needs_copy(dst_vma, src_vma))
                return 0;

        if (is_vm_hugetlb_page(src_vma))
                return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);

        /*
         * We need to invalidate the secondary MMU mappings only when
         * there could be a permission downgrade on the ptes of the
         * parent mm. And a permission downgrade will only happen if
         * is_cow_mapping() returns true.
         */
        is_cow = is_cow_mapping(src_vma->vm_flags);

        if (is_cow) {
                mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
                                        0, src_mm, addr, end);
                mmu_notifier_invalidate_range_start(&range);
                /*
                 * Disabling preemption is not needed for the write side, as
                 * the read side doesn't spin, but goes to the mmap_lock.
                 *
                 * Use the raw variant of the seqcount_t write API to avoid
                 * lockdep complaining about preemptibility.
                 */
                vma_assert_write_locked(src_vma);
                raw_write_seqcount_begin(&src_mm->write_protect_seq);
        }

        ret = 0;
        dst_pgd = pgd_offset(dst_mm, addr);
        src_pgd = pgd_offset(src_mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(src_pgd))
                        continue;
                if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
                                            addr, next))) {
                        ret = -ENOMEM;
                        break;
                }
        } while (dst_pgd++, src_pgd++, addr = next, addr != end);

        if (is_cow) {
                raw_write_seqcount_end(&src_mm->write_protect_seq);
                mmu_notifier_invalidate_range_end(&range);
        }
        return ret;
}

/* Whether we should zap all COWed (private) pages too */
static inline bool should_zap_cows(struct zap_details *details)
{
        /* By default, zap all pages */
        if (!details || details->reclaim_pt)
                return true;

        /* Or, we zap COWed pages only if the caller wants to */
        return details->even_cows;
}

/* Decides whether we should zap this folio with the folio pointer specified */
static inline bool should_zap_folio(struct zap_details *details,
                                    struct folio *folio)
{
        /* If we can make a decision without *folio.. */
        if (should_zap_cows(details))
                return true;

        /* Otherwise we should only zap non-anon folios */
        return !folio_test_anon(folio);
}

static inline bool zap_drop_markers(struct zap_details *details)
{
        if (!details)
                return false;

        return details->zap_flags & ZAP_FLAG_DROP_MARKER;
}

/*
 * This function makes sure that we'll replace the none pte with an uffd-wp
 * swap special pte marker when necessary. Must be with the pgtable lock held.
 *
 * Returns true if uffd-wp ptes was installed, false otherwise.
 */
static inline bool
zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
                              unsigned long addr, pte_t *pte, int nr,
                              struct zap_details *details, pte_t pteval)
{
        bool was_installed = false;

#ifdef CONFIG_PTE_MARKER_UFFD_WP
        /* Zap on anonymous always means dropping everything */
        if (vma_is_anonymous(vma))
                return false;

        if (zap_drop_markers(details))
                return false;

        for (;;) {
                /* the PFN in the PTE is irrelevant. */
                if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval))
                        was_installed = true;
                if (--nr == 0)
                        break;
                pte++;
                addr += PAGE_SIZE;
        }
#endif
        return was_installed;
}

static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,
                struct vm_area_struct *vma, struct folio *folio,
                struct page *page, pte_t *pte, pte_t ptent, unsigned int nr,
                unsigned long addr, struct zap_details *details, int *rss,
                bool *force_flush, bool *force_break, bool *any_skipped)
{
        struct mm_struct *mm = tlb->mm;
        bool delay_rmap = false;

        if (!folio_test_anon(folio)) {
                ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
                if (pte_dirty(ptent)) {
                        folio_mark_dirty(folio);
                        if (tlb_delay_rmap(tlb)) {
                                delay_rmap = true;
                                *force_flush = true;
                        }
                }
                if (pte_young(ptent) && likely(vma_has_recency(vma)))
                        folio_mark_accessed(folio);
                rss[mm_counter(folio)] -= nr;
        } else {
                /* We don't need up-to-date accessed/dirty bits. */
                clear_full_ptes(mm, addr, pte, nr, tlb->fullmm);
                rss[MM_ANONPAGES] -= nr;
        }
        /* Checking a single PTE in a batch is sufficient. */
        arch_check_zapped_pte(vma, ptent);
        tlb_remove_tlb_entries(tlb, pte, nr, addr);
        if (unlikely(userfaultfd_pte_wp(vma, ptent)))
                *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte,
                                                             nr, details, ptent);

        if (!delay_rmap) {
                folio_remove_rmap_ptes(folio, page, nr, vma);

                if (unlikely(folio_mapcount(folio) < 0))
                        print_bad_pte(vma, addr, ptent, page);
        }
        if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) {
                *force_flush = true;
                *force_break = true;
        }
}

/*
 * Zap or skip at least one present PTE, trying to batch-process subsequent
 * PTEs that map consecutive pages of the same folio.
 *
 * Returns the number of processed (skipped or zapped) PTEs (at least 1).
 */
static inline int zap_present_ptes(struct mmu_gather *tlb,
                struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
                unsigned int max_nr, unsigned long addr,
                struct zap_details *details, int *rss, bool *force_flush,
                bool *force_break, bool *any_skipped)
{
        const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        struct mm_struct *mm = tlb->mm;
        struct folio *folio;
        struct page *page;
        int nr;

        page = vm_normal_page(vma, addr, ptent);
        if (!page) {
                /* We don't need up-to-date accessed/dirty bits. */
                ptep_get_and_clear_full(mm, addr, pte, tlb->fullmm);
                arch_check_zapped_pte(vma, ptent);
                tlb_remove_tlb_entry(tlb, pte, addr);
                if (userfaultfd_pte_wp(vma, ptent))
                        *any_skipped = zap_install_uffd_wp_if_needed(vma, addr,
                                                pte, 1, details, ptent);
                ksm_might_unmap_zero_page(mm, ptent);
                return 1;
        }

        folio = page_folio(page);
        if (unlikely(!should_zap_folio(details, folio))) {
                *any_skipped = true;
                return 1;
        }

        /*
         * Make sure that the common "small folio" case is as fast as possible
         * by keeping the batching logic separate.
         */
        if (unlikely(folio_test_large(folio) && max_nr != 1)) {
                nr = folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags,
                                     NULL, NULL, NULL);

                zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr,
                                       addr, details, rss, force_flush,
                                       force_break, any_skipped);
                return nr;
        }
        zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, 1, addr,
                               details, rss, force_flush, force_break, any_skipped);
        return 1;
}

static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
                struct vm_area_struct *vma, pte_t *pte, pte_t ptent,
                unsigned int max_nr, unsigned long addr,
                struct zap_details *details, int *rss, bool *any_skipped)
{
        swp_entry_t entry;
        int nr = 1;

        *any_skipped = true;
        entry = pte_to_swp_entry(ptent);
        if (is_device_private_entry(entry) ||
                is_device_exclusive_entry(entry)) {
                struct page *page = pfn_swap_entry_to_page(entry);
                struct folio *folio = page_folio(page);

                if (unlikely(!should_zap_folio(details, folio)))
                        return 1;
                /*
                 * Both device private/exclusive mappings should only
                 * work with anonymous page so far, so we don't need to
                 * consider uffd-wp bit when zap. For more information,
                 * see zap_install_uffd_wp_if_needed().
                 */
                WARN_ON_ONCE(!vma_is_anonymous(vma));
                rss[mm_counter(folio)]--;
                folio_remove_rmap_pte(folio, page, vma);
                folio_put(folio);
        } else if (!non_swap_entry(entry)) {
                /* Genuine swap entries, hence a private anon pages */
                if (!should_zap_cows(details))
                        return 1;

                nr = swap_pte_batch(pte, max_nr, ptent);
                rss[MM_SWAPENTS] -= nr;
                free_swap_and_cache_nr(entry, nr);
        } else if (is_migration_entry(entry)) {
                struct folio *folio = pfn_swap_entry_folio(entry);

                if (!should_zap_folio(details, folio))
                        return 1;
                rss[mm_counter(folio)]--;
        } else if (pte_marker_entry_uffd_wp(entry)) {
                /*
                 * For anon: always drop the marker; for file: only
                 * drop the marker if explicitly requested.
                 */
                if (!vma_is_anonymous(vma) && !zap_drop_markers(details))
                        return 1;
        } else if (is_guard_swp_entry(entry)) {
                /*
                 * Ordinary zapping should not remove guard PTE
                 * markers. Only do so if we should remove PTE markers
                 * in general.
                 */
                if (!zap_drop_markers(details))
                        return 1;
        } else if (is_hwpoison_entry(entry) || is_poisoned_swp_entry(entry)) {
                if (!should_zap_cows(details))
                        return 1;
        } else {
                /* We should have covered all the swap entry types */
                pr_alert("unrecognized swap entry 0x%lx\n", entry.val);
                WARN_ON_ONCE(1);
        }
        clear_not_present_full_ptes(vma->vm_mm, addr, pte, nr, tlb->fullmm);
        *any_skipped = zap_install_uffd_wp_if_needed(vma, addr, pte, nr, details, ptent);

        return nr;
}

static inline int do_zap_pte_range(struct mmu_gather *tlb,
                                   struct vm_area_struct *vma, pte_t *pte,
                                   unsigned long addr, unsigned long end,
                                   struct zap_details *details, int *rss,
                                   bool *force_flush, bool *force_break,
                                   bool *any_skipped)
{
        pte_t ptent = ptep_get(pte);
        int max_nr = (end - addr) / PAGE_SIZE;
        int nr = 0;

        /* Skip all consecutive none ptes */
        if (pte_none(ptent)) {
                for (nr = 1; nr < max_nr; nr++) {
                        ptent = ptep_get(pte + nr);
                        if (!pte_none(ptent))
                                break;
                }
                max_nr -= nr;
                if (!max_nr)
                        return nr;
                pte += nr;
                addr += nr * PAGE_SIZE;
        }

        if (pte_present(ptent))
                nr += zap_present_ptes(tlb, vma, pte, ptent, max_nr, addr,
                                       details, rss, force_flush, force_break,
                                       any_skipped);
        else
                nr += zap_nonpresent_ptes(tlb, vma, pte, ptent, max_nr, addr,
                                          details, rss, any_skipped);

        return nr;
}

static unsigned long zap_pte_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pmd_t *pmd,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        bool force_flush = false, force_break = false;
        struct mm_struct *mm = tlb->mm;
        int rss[NR_MM_COUNTERS];
        spinlock_t *ptl;
        pte_t *start_pte;
        pte_t *pte;
        pmd_t pmdval;
        unsigned long start = addr;
        bool can_reclaim_pt = reclaim_pt_is_enabled(start, end, details);
        bool direct_reclaim = true;
        int nr;

retry:
        tlb_change_page_size(tlb, PAGE_SIZE);
        init_rss_vec(rss);
        start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
        if (!pte)
                return addr;

        flush_tlb_batched_pending(mm);
        arch_enter_lazy_mmu_mode();
        do {
                bool any_skipped = false;

                if (need_resched()) {
                        direct_reclaim = false;
                        break;
                }

                nr = do_zap_pte_range(tlb, vma, pte, addr, end, details, rss,
                                      &force_flush, &force_break, &any_skipped);
                if (any_skipped)
                        can_reclaim_pt = false;
                if (unlikely(force_break)) {
                        addr += nr * PAGE_SIZE;
                        direct_reclaim = false;
                        break;
                }
        } while (pte += nr, addr += PAGE_SIZE * nr, addr != end);

        /*
         * Fast path: try to hold the pmd lock and unmap the PTE page.
         *
         * If the pte lock was released midway (retry case), or if the attempt
         * to hold the pmd lock failed, then we need to recheck all pte entries
         * to ensure they are still none, thereby preventing the pte entries
         * from being repopulated by another thread.
         */
        if (can_reclaim_pt && direct_reclaim && addr == end)
                direct_reclaim = try_get_and_clear_pmd(mm, pmd, &pmdval);

        add_mm_rss_vec(mm, rss);
        arch_leave_lazy_mmu_mode();

        /* Do the actual TLB flush before dropping ptl */
        if (force_flush) {
                tlb_flush_mmu_tlbonly(tlb);
                tlb_flush_rmaps(tlb, vma);
        }
        pte_unmap_unlock(start_pte, ptl);

        /*
         * If we forced a TLB flush (either due to running out of
         * batch buffers or because we needed to flush dirty TLB
         * entries before releasing the ptl), free the batched
         * memory too. Come back again if we didn't do everything.
         */
        if (force_flush)
                tlb_flush_mmu(tlb);

        if (addr != end) {
                cond_resched();
                force_flush = false;
                force_break = false;
                goto retry;
        }

        if (can_reclaim_pt) {
                if (direct_reclaim)
                        free_pte(mm, start, tlb, pmdval);
                else
                        try_to_free_pte(mm, pmd, start, tlb);
        }

        return addr;
}

static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pud_t *pud,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        pmd_t *pmd;
        unsigned long next;

        pmd = pmd_offset(pud, addr);
        do {
                next = pmd_addr_end(addr, end);
                if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
                        if (next - addr != HPAGE_PMD_SIZE)
                                __split_huge_pmd(vma, pmd, addr, false);
                        else if (zap_huge_pmd(tlb, vma, pmd, addr)) {
                                addr = next;
                                continue;
                        }
                        /* fall through */
                } else if (details && details->single_folio &&
                           folio_test_pmd_mappable(details->single_folio) &&
                           next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) {
                        spinlock_t *ptl = pmd_lock(tlb->mm, pmd);
                        /*
                         * Take and drop THP pmd lock so that we cannot return
                         * prematurely, while zap_huge_pmd() has cleared *pmd,
                         * but not yet decremented compound_mapcount().
                         */
                        spin_unlock(ptl);
                }
                if (pmd_none(*pmd)) {
                        addr = next;
                        continue;
                }
                addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
                if (addr != next)
                        pmd--;
        } while (pmd++, cond_resched(), addr != end);

        return addr;
}

static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, p4d_t *p4d,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        pud_t *pud;
        unsigned long next;

        pud = pud_offset(p4d, addr);
        do {
                next = pud_addr_end(addr, end);
                if (pud_trans_huge(*pud) || pud_devmap(*pud)) {
                        if (next - addr != HPAGE_PUD_SIZE) {
                                mmap_assert_locked(tlb->mm);
                                split_huge_pud(vma, pud, addr);
                        } else if (zap_huge_pud(tlb, vma, pud, addr))
                                goto next;
                        /* fall through */
                }
                if (pud_none_or_clear_bad(pud))
                        continue;
                next = zap_pmd_range(tlb, vma, pud, addr, next, details);
next:
                cond_resched();
        } while (pud++, addr = next, addr != end);

        return addr;
}

static inline unsigned long zap_p4d_range(struct mmu_gather *tlb,
                                struct vm_area_struct *vma, pgd_t *pgd,
                                unsigned long addr, unsigned long end,
                                struct zap_details *details)
{
        p4d_t *p4d;
        unsigned long next;

        p4d = p4d_offset(pgd, addr);
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none_or_clear_bad(p4d))
                        continue;
                next = zap_pud_range(tlb, vma, p4d, addr, next, details);
        } while (p4d++, addr = next, addr != end);

        return addr;
}

void unmap_page_range(struct mmu_gather *tlb,
                             struct vm_area_struct *vma,
                             unsigned long addr, unsigned long end,
                             struct zap_details *details)
{
        pgd_t *pgd;
        unsigned long next;

        BUG_ON(addr >= end);
        tlb_start_vma(tlb, vma);
        pgd = pgd_offset(vma->vm_mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none_or_clear_bad(pgd))
                        continue;
                next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
        } while (pgd++, addr = next, addr != end);
        tlb_end_vma(tlb, vma);
}


static void unmap_single_vma(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr,
                struct zap_details *details, bool mm_wr_locked)
{
        unsigned long start = max(vma->vm_start, start_addr);
        unsigned long end;

        if (start >= vma->vm_end)
                return;
        end = min(vma->vm_end, end_addr);
        if (end <= vma->vm_start)
                return;

        if (vma->vm_file)
                uprobe_munmap(vma, start, end);

        if (start != end) {
                if (unlikely(is_vm_hugetlb_page(vma))) {
                        /*
                         * It is undesirable to test vma->vm_file as it
                         * should be non-null for valid hugetlb area.
                         * However, vm_file will be NULL in the error
                         * cleanup path of mmap_region. When
                         * hugetlbfs ->mmap method fails,
                         * mmap_region() nullifies vma->vm_file
                         * before calling this function to clean up.
                         * Since no pte has actually been setup, it is
                         * safe to do nothing in this case.
                         */
                        if (vma->vm_file) {
                                zap_flags_t zap_flags = details ?
                                    details->zap_flags : 0;
                                __unmap_hugepage_range(tlb, vma, start, end,
                                                             NULL, zap_flags);
                        }
                } else
                        unmap_page_range(tlb, vma, start, end, details);
        }
}

/**
 * unmap_vmas - unmap a range of memory covered by a list of vma's
 * @tlb: address of the caller's struct mmu_gather
 * @mas: the maple state
 * @vma: the starting vma
 * @start_addr: virtual address at which to start unmapping
 * @end_addr: virtual address at which to end unmapping
 * @tree_end: The maximum index to check
 * @mm_wr_locked: lock flag
 *
 * Unmap all pages in the vma list.
 *
 * Only addresses between `start' and `end' will be unmapped.
 *
 * The VMA list must be sorted in ascending virtual address order.
 *
 * unmap_vmas() assumes that the caller will flush the whole unmapped address
 * range after unmap_vmas() returns.  So the only responsibility here is to
 * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
 * drops the lock and schedules.
 */
void unmap_vmas(struct mmu_gather *tlb, struct ma_state *mas,
                struct vm_area_struct *vma, unsigned long start_addr,
                unsigned long end_addr, unsigned long tree_end,
                bool mm_wr_locked)
{
        struct mmu_notifier_range range;
        struct zap_details details = {
                .zap_flags = ZAP_FLAG_DROP_MARKER | ZAP_FLAG_UNMAP,
                /* Careful - we need to zap private pages too! */
                .even_cows = true,
        };

        mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma->vm_mm,
                                start_addr, end_addr);
        mmu_notifier_invalidate_range_start(&range);
        do {
                unsigned long start = start_addr;
                unsigned long end = end_addr;
                hugetlb_zap_begin(vma, &start, &end);
                unmap_single_vma(tlb, vma, start, end, &details,
                                 mm_wr_locked);
                hugetlb_zap_end(vma, &details);
                vma = mas_find(mas, tree_end - 1);
        } while (vma && likely(!xa_is_zero(vma)));
        mmu_notifier_invalidate_range_end(&range);
}

/**
 * zap_page_range_single_batched - remove user pages in a given range
 * @tlb: pointer to the caller's struct mmu_gather
 * @vma: vm_area_struct holding the applicable pages
 * @address: starting address of pages to remove
 * @size: number of bytes to remove
 * @details: details of shared cache invalidation
 *
 * @tlb shouldn't be NULL.  The range must fit into one VMA.  If @vma is for
 * hugetlb, @tlb is flushed and re-initialized by this function.
 */
void zap_page_range_single_batched(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long address,
                unsigned long size, struct zap_details *details)
{
        const unsigned long end = address + size;
        struct mmu_notifier_range range;

        VM_WARN_ON_ONCE(!tlb || tlb->mm != vma->vm_mm);

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
                                address, end);
        hugetlb_zap_begin(vma, &range.start, &range.end);
        update_hiwater_rss(vma->vm_mm);
        mmu_notifier_invalidate_range_start(&range);
        /*
         * unmap 'address-end' not 'range.start-range.end' as range
         * could have been expanded for hugetlb pmd sharing.
         */
        unmap_single_vma(tlb, vma, address, end, details, false);
        mmu_notifier_invalidate_range_end(&range);
        if (is_vm_hugetlb_page(vma)) {
                /*
                 * flush tlb and free resources before hugetlb_zap_end(), to
                 * avoid concurrent page faults' allocation failure.
                 */
                tlb_finish_mmu(tlb);
                hugetlb_zap_end(vma, details);
                tlb_gather_mmu(tlb, vma->vm_mm);
        }
}

/**
 * zap_page_range_single - remove user pages in a given range
 * @vma: vm_area_struct holding the applicable pages
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
 * @details: details of shared cache invalidation
 *
 * The range must fit into one VMA.
 */
void zap_page_range_single(struct vm_area_struct *vma, unsigned long address,
                unsigned long size, struct zap_details *details)
{
        struct mmu_gather tlb;

        tlb_gather_mmu(&tlb, vma->vm_mm);
        zap_page_range_single_batched(&tlb, vma, address, size, details);
        tlb_finish_mmu(&tlb);
}

/**
 * zap_vma_ptes - remove ptes mapping the vma
 * @vma: vm_area_struct holding ptes to be zapped
 * @address: starting address of pages to zap
 * @size: number of bytes to zap
 *
 * This function only unmaps ptes assigned to VM_PFNMAP vmas.
 *
 * The entire address range must be fully contained within the vma.
 *
 */
void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
                unsigned long size)
{
        if (!range_in_vma(vma, address, address + size) ||
                            !(vma->vm_flags & VM_PFNMAP))
                return;

        zap_page_range_single(vma, address, size, NULL);
}
EXPORT_SYMBOL_GPL(zap_vma_ptes);

static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr)
{
        pgd_t *pgd;
        p4d_t *p4d;
        pud_t *pud;
        pmd_t *pmd;

        pgd = pgd_offset(mm, addr);
        p4d = p4d_alloc(mm, pgd, addr);
        if (!p4d)
                return NULL;
        pud = pud_alloc(mm, p4d, addr);
        if (!pud)
                return NULL;
        pmd = pmd_alloc(mm, pud, addr);
        if (!pmd)
                return NULL;

        VM_BUG_ON(pmd_trans_huge(*pmd));
        return pmd;
}

pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
                        spinlock_t **ptl)
{
        pmd_t *pmd = walk_to_pmd(mm, addr);

        if (!pmd)
                return NULL;
        return pte_alloc_map_lock(mm, pmd, addr, ptl);
}

static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
{
        VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP);
        /*
         * Whoever wants to forbid the zeropage after some zeropages
         * might already have been mapped has to scan the page tables and
         * bail out on any zeropages. Zeropages in COW mappings can
         * be unshared using FAULT_FLAG_UNSHARE faults.
         */
        if (mm_forbids_zeropage(vma->vm_mm))
                return false;
        /* zeropages in COW mappings are common and unproblematic. */
        if (is_cow_mapping(vma->vm_flags))
                return true;
        /* Mappings that do not allow for writable PTEs are unproblematic. */
        if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
                return true;
        /*
         * Why not allow any VMA that has vm_ops->pfn_mkwrite? GUP could
         * find the shared zeropage and longterm-pin it, which would
         * be problematic as soon as the zeropage gets replaced by a different
         * page due to vma->vm_ops->pfn_mkwrite, because what's mapped would
         * now differ to what GUP looked up. FSDAX is incompatible to
         * FOLL_LONGTERM and VM_IO is incompatible to GUP completely (see
         * check_vma_flags).
         */
        return vma->vm_ops && vma->vm_ops->pfn_mkwrite &&
               (vma_is_fsdax(vma) || vma->vm_flags & VM_IO);
}

static int validate_page_before_insert(struct vm_area_struct *vma,
                                       struct page *page)
{
        struct folio *folio = page_folio(page);

        if (!folio_ref_count(folio))
                return -EINVAL;
        if (unlikely(is_zero_folio(folio))) {
                if (!vm_mixed_zeropage_allowed(vma))
                        return -EINVAL;
                return 0;
        }
        if (folio_test_anon(folio) || folio_test_slab(folio) ||
            page_has_type(page))
                return -EINVAL;
        flush_dcache_folio(folio);
        return 0;
}

static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte,
                                unsigned long addr, struct page *page,
                                pgprot_t prot, bool mkwrite)
{
        struct folio *folio = page_folio(page);
        pte_t pteval = ptep_get(pte);

        if (!pte_none(pteval)) {
                if (!mkwrite)
                        return -EBUSY;

                /* see insert_pfn(). */
                if (pte_pfn(pteval) != page_to_pfn(page)) {
                        WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval)));
                        return -EFAULT;
                }
                pteval = maybe_mkwrite(pteval, vma);
                pteval = pte_mkyoung(pteval);
                if (ptep_set_access_flags(vma, addr, pte, pteval, 1))
                        update_mmu_cache(vma, addr, pte);
                return 0;
        }

        /* Ok, finally just insert the thing.. */
        pteval = mk_pte(page, prot);
        if (unlikely(is_zero_folio(folio))) {
                pteval = pte_mkspecial(pteval);
        } else {
                folio_get(folio);
                pteval = mk_pte(page, prot);
                if (mkwrite) {
                        pteval = pte_mkyoung(pteval);
                        pteval = maybe_mkwrite(pte_mkdirty(pteval), vma);
                }
                inc_mm_counter(vma->vm_mm, mm_counter_file(folio));
                folio_add_file_rmap_pte(folio, page, vma);
        }
        set_pte_at(vma->vm_mm, addr, pte, pteval);
        return 0;
}

static int insert_page(struct vm_area_struct *vma, unsigned long addr,
                        struct page *page, pgprot_t prot, bool mkwrite)
{
        int retval;
        pte_t *pte;
        spinlock_t *ptl;

        retval = validate_page_before_insert(vma, page);
        if (retval)
                goto out;
        retval = -ENOMEM;
        pte = get_locked_pte(vma->vm_mm, addr, &ptl);
        if (!pte)
                goto out;
        retval = insert_page_into_pte_locked(vma, pte, addr, page, prot,
                                        mkwrite);
        pte_unmap_unlock(pte, ptl);
out:
        return retval;
}

static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte,
                        unsigned long addr, struct page *page, pgprot_t prot)
{
        int err;

        err = validate_page_before_insert(vma, page);
        if (err)
                return err;
        return insert_page_into_pte_locked(vma, pte, addr, page, prot, false);
}

/* insert_pages() amortizes the cost of spinlock operations
 * when inserting pages in a loop.
 */
static int insert_pages(struct vm_area_struct *vma, unsigned long addr,
                        struct page **pages, unsigned long *num, pgprot_t prot)
{
        pmd_t *pmd = NULL;
        pte_t *start_pte, *pte;
        spinlock_t *pte_lock;
        struct mm_struct *const mm = vma->vm_mm;
        unsigned long curr_page_idx = 0;
        unsigned long remaining_pages_total = *num;
        unsigned long pages_to_write_in_pmd;
        int ret;
more:
        ret = -EFAULT;
        pmd = walk_to_pmd(mm, addr);
        if (!pmd)
                goto out;

        pages_to_write_in_pmd = min_t(unsigned long,
                remaining_pages_total, PTRS_PER_PTE - pte_index(addr));

        /* Allocate the PTE if necessary; takes PMD lock once only. */
        ret = -ENOMEM;
        if (pte_alloc(mm, pmd))
                goto out;

        while (pages_to_write_in_pmd) {
                int pte_idx = 0;
                const int batch_size = min_t(int, pages_to_write_in_pmd, 8);

                start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock);
                if (!start_pte) {
                        ret = -EFAULT;
                        goto out;
                }
                for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) {
                        int err = insert_page_in_batch_locked(vma, pte,
                                addr, pages[curr_page_idx], prot);
                        if (unlikely(err)) {
                                pte_unmap_unlock(start_pte, pte_lock);
                                ret = err;
                                remaining_pages_total -= pte_idx;
                                goto out;
                        }
                        addr += PAGE_SIZE;
                        ++curr_page_idx;
                }
                pte_unmap_unlock(start_pte, pte_lock);
                pages_to_write_in_pmd -= batch_size;
                remaining_pages_total -= batch_size;
        }
        if (remaining_pages_total)
                goto more;
        ret = 0;
out:
        *num = remaining_pages_total;
        return ret;
}

/**
 * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock.
 * @vma: user vma to map to
 * @addr: target start user address of these pages
 * @pages: source kernel pages
 * @num: in: number of pages to map. out: number of pages that were *not*
 * mapped. (0 means all pages were successfully mapped).
 *
 * Preferred over vm_insert_page() when inserting multiple pages.
 *
 * In case of error, we may have mapped a subset of the provided
 * pages. It is the caller's responsibility to account for this case.
 *
 * The same restrictions apply as in vm_insert_page().
 */
int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr,
                        struct page **pages, unsigned long *num)
{
        const unsigned long end_addr = addr + (*num * PAGE_SIZE) - 1;

        if (addr < vma->vm_start || end_addr >= vma->vm_end)
                return -EFAULT;
        if (!(vma->vm_flags & VM_MIXEDMAP)) {
                BUG_ON(mmap_read_trylock(vma->vm_mm));
                BUG_ON(vma->vm_flags & VM_PFNMAP);
                vm_flags_set(vma, VM_MIXEDMAP);
        }
        /* Defer page refcount checking till we're about to map that page. */
        return insert_pages(vma, addr, pages, num, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_insert_pages);

/**
 * vm_insert_page - insert single page into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @page: source kernel page
 *
 * This allows drivers to insert individual pages they've allocated
 * into a user vma. The zeropage is supported in some VMAs,
 * see vm_mixed_zeropage_allowed().
 *
 * The page has to be a nice clean _individual_ kernel allocation.
 * If you allocate a compound page, you need to have marked it as
 * such (__GFP_COMP), or manually just split the page up yourself
 * (see split_page()).
 *
 * NOTE! Traditionally this was done with "remap_pfn_range()" which
 * took an arbitrary page protection parameter. This doesn't allow
 * that. Your vma protection will have to be set up correctly, which
 * means that if you want a shared writable mapping, you'd better
 * ask for a shared writable mapping!
 *
 * The page does not need to be reserved.
 *
 * Usually this function is called from f_op->mmap() handler
 * under mm->mmap_lock write-lock, so it can change vma->vm_flags.
 * Caller must set VM_MIXEDMAP on vma if it wants to call this
 * function from other places, for example from page-fault handler.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
                        struct page *page)
{
        if (addr < vma->vm_start || addr >= vma->vm_end)
                return -EFAULT;
        if (!(vma->vm_flags & VM_MIXEDMAP)) {
                BUG_ON(mmap_read_trylock(vma->vm_mm));
                BUG_ON(vma->vm_flags & VM_PFNMAP);
                vm_flags_set(vma, VM_MIXEDMAP);
        }
        return insert_page(vma, addr, page, vma->vm_page_prot, false);
}
EXPORT_SYMBOL(vm_insert_page);

/*
 * __vm_map_pages - maps range of kernel pages into user vma
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 * @offset: user's requested vm_pgoff
 *
 * This allows drivers to map range of kernel pages into a user vma.
 * The zeropage is supported in some VMAs, see
 * vm_mixed_zeropage_allowed().
 *
 * Return: 0 on success and error code otherwise.
 */
static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num, unsigned long offset)
{
        unsigned long count = vma_pages(vma);
        unsigned long uaddr = vma->vm_start;
        int ret, i;

        /* Fail if the user requested offset is beyond the end of the object */
        if (offset >= num)
                return -ENXIO;

        /* Fail if the user requested size exceeds available object size */
        if (count > num - offset)
                return -ENXIO;

        for (i = 0; i < count; i++) {
                ret = vm_insert_page(vma, uaddr, pages[offset + i]);
                if (ret < 0)
                        return ret;
                uaddr += PAGE_SIZE;
        }

        return 0;
}

/**
 * vm_map_pages - maps range of kernel pages starts with non zero offset
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 *
 * Maps an object consisting of @num pages, catering for the user's
 * requested vm_pgoff
 *
 * If we fail to insert any page into the vma, the function will return
 * immediately leaving any previously inserted pages present.  Callers
 * from the mmap handler may immediately return the error as their caller
 * will destroy the vma, removing any successfully inserted pages. Other
 * callers should make their own arrangements for calling unmap_region().
 *
 * Context: Process context. Called by mmap handlers.
 * Return: 0 on success and error code otherwise.
 */
int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num)
{
        return __vm_map_pages(vma, pages, num, vma->vm_pgoff);
}
EXPORT_SYMBOL(vm_map_pages);

/**
 * vm_map_pages_zero - map range of kernel pages starts with zero offset
 * @vma: user vma to map to
 * @pages: pointer to array of source kernel pages
 * @num: number of pages in page array
 *
 * Similar to vm_map_pages(), except that it explicitly sets the offset
 * to 0. This function is intended for the drivers that did not consider
 * vm_pgoff.
 *
 * Context: Process context. Called by mmap handlers.
 * Return: 0 on success and error code otherwise.
 */
int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
                                unsigned long num)
{
        return __vm_map_pages(vma, pages, num, 0);
}
EXPORT_SYMBOL(vm_map_pages_zero);

static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                        pfn_t pfn, pgprot_t prot, bool mkwrite)
{
        struct mm_struct *mm = vma->vm_mm;
        pte_t *pte, entry;
        spinlock_t *ptl;

        pte = get_locked_pte(mm, addr, &ptl);
        if (!pte)
                return VM_FAULT_OOM;
        entry = ptep_get(pte);
        if (!pte_none(entry)) {
                if (mkwrite) {
                        /*
                         * For read faults on private mappings the PFN passed
                         * in may not match the PFN we have mapped if the
                         * mapped PFN is a writeable COW page.  In the mkwrite
                         * case we are creating a writable PTE for a shared
                         * mapping and we expect the PFNs to match. If they
                         * don't match, we are likely racing with block
                         * allocation and mapping invalidation so just skip the
                         * update.
                         */
                        if (pte_pfn(entry) != pfn_t_to_pfn(pfn)) {
                                WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry)));
                                goto out_unlock;
                        }
                        entry = pte_mkyoung(entry);
                        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                        if (ptep_set_access_flags(vma, addr, pte, entry, 1))
                                update_mmu_cache(vma, addr, pte);
                }
                goto out_unlock;
        }

        /* Ok, finally just insert the thing.. */
        if (pfn_t_devmap(pfn))
                entry = pte_mkdevmap(pfn_t_pte(pfn, prot));
        else
                entry = pte_mkspecial(pfn_t_pte(pfn, prot));

        if (mkwrite) {
                entry = pte_mkyoung(entry);
                entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        }

        set_pte_at(mm, addr, pte, entry);
        update_mmu_cache(vma, addr, pte); /* XXX: why not for insert_page? */

out_unlock:
        pte_unmap_unlock(pte, ptl);
        return VM_FAULT_NOPAGE;
}

/**
 * vmf_insert_pfn_prot - insert single pfn into user vma with specified pgprot
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 * @pgprot: pgprot flags for the inserted page
 *
 * This is exactly like vmf_insert_pfn(), except that it allows drivers
 * to override pgprot on a per-page basis.
 *
 * This only makes sense for IO mappings, and it makes no sense for
 * COW mappings.  In general, using multiple vmas is preferable;
 * vmf_insert_pfn_prot should only be used if using multiple VMAs is
 * impractical.
 *
 * pgprot typically only differs from @vma->vm_page_prot when drivers set
 * caching- and encryption bits different than those of @vma->vm_page_prot,
 * because the caching- or encryption mode may not be known at mmap() time.
 *
 * This is ok as long as @vma->vm_page_prot is not used by the core vm
 * to set caching and encryption bits for those vmas (except for COW pages).
 * This is ensured by core vm only modifying these page table entries using
 * functions that don't touch caching- or encryption bits, using pte_modify()
 * if needed. (See for example mprotect()).
 *
 * Also when new page-table entries are created, this is only done using the
 * fault() callback, and never using the value of vma->vm_page_prot,
 * except for page-table entries that point to anonymous pages as the result
 * of COW.
 *
 * Context: Process context.  May allocate using %GFP_KERNEL.
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
                        unsigned long pfn, pgprot_t pgprot)
{
        /*
         * Technically, architectures with pte_special can avoid all these
         * restrictions (same for remap_pfn_range).  However we would like
         * consistency in testing and feature parity among all, so we should
         * try to keep these invariants in place for everybody.
         */
        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
                                                (VM_PFNMAP|VM_MIXEDMAP));
        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
        BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        if (!pfn_modify_allowed(pfn, pgprot))
                return VM_FAULT_SIGBUS;

        pfnmap_setup_cachemode_pfn(pfn, &pgprot);

        return insert_pfn(vma, addr, __pfn_to_pfn_t(pfn, PFN_DEV), pgprot,
                        false);
}
EXPORT_SYMBOL(vmf_insert_pfn_prot);

/**
 * vmf_insert_pfn - insert single pfn into user vma
 * @vma: user vma to map to
 * @addr: target user address of this page
 * @pfn: source kernel pfn
 *
 * Similar to vm_insert_page, this allows drivers to insert individual pages
 * they've allocated into a user vma. Same comments apply.
 *
 * This function should only be called from a vm_ops->fault handler, and
 * in that case the handler should return the result of this function.
 *
 * vma cannot be a COW mapping.
 *
 * As this is called only for pages that do not currently exist, we
 * do not need to flush old virtual caches or the TLB.
 *
 * Context: Process context.  May allocate using %GFP_KERNEL.
 * Return: vm_fault_t value.
 */
vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
                        unsigned long pfn)
{
        return vmf_insert_pfn_prot(vma, addr, pfn, vma->vm_page_prot);
}
EXPORT_SYMBOL(vmf_insert_pfn);

static bool vm_mixed_ok(struct vm_area_struct *vma, pfn_t pfn, bool mkwrite)
{
        if (unlikely(is_zero_pfn(pfn_t_to_pfn(pfn))) &&
            (mkwrite || !vm_mixed_zeropage_allowed(vma)))
                return false;
        /* these checks mirror the abort conditions in vm_normal_page */
        if (vma->vm_flags & VM_MIXEDMAP)
                return true;
        if (pfn_t_devmap(pfn))
                return true;
        if (pfn_t_special(pfn))
                return true;
        if (is_zero_pfn(pfn_t_to_pfn(pfn)))
                return true;
        return false;
}

static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma,
                unsigned long addr, pfn_t pfn, bool mkwrite)
{
        pgprot_t pgprot = vma->vm_page_prot;
        int err;

        if (!vm_mixed_ok(vma, pfn, mkwrite))
                return VM_FAULT_SIGBUS;

        if (addr < vma->vm_start || addr >= vma->vm_end)
                return VM_FAULT_SIGBUS;

        pfnmap_setup_cachemode_pfn(pfn_t_to_pfn(pfn), &pgprot);

        if (!pfn_modify_allowed(pfn_t_to_pfn(pfn), pgprot))
                return VM_FAULT_SIGBUS;

        /*
         * If we don't have pte special, then we have to use the pfn_valid()
         * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
         * refcount the page if pfn_valid is true (hence insert_page rather
         * than insert_pfn).  If a zero_pfn were inserted into a VM_MIXEDMAP
         * without pte special, it would there be refcounted as a normal page.
         */
        if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL) &&
            !pfn_t_devmap(pfn) && pfn_t_valid(pfn)) {
                struct page *page;

                /*
                 * At this point we are committed to insert_page()
                 * regardless of whether the caller specified flags that
                 * result in pfn_t_has_page() == false.
                 */
                page = pfn_to_page(pfn_t_to_pfn(pfn));
                err = insert_page(vma, addr, page, pgprot, mkwrite);
        } else {
                return insert_pfn(vma, addr, pfn, pgprot, mkwrite);
        }

        if (err == -ENOMEM)
                return VM_FAULT_OOM;
        if (err < 0 && err != -EBUSY)
                return VM_FAULT_SIGBUS;

        return VM_FAULT_NOPAGE;
}

vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page,
                        bool write)
{
        pgprot_t pgprot = vmf->vma->vm_page_prot;
        unsigned long addr = vmf->address;
        int err;

        if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end)
                return VM_FAULT_SIGBUS;

        err = insert_page(vmf->vma, addr, page, pgprot, write);
        if (err == -ENOMEM)
                return VM_FAULT_OOM;
        if (err < 0 && err != -EBUSY)
                return VM_FAULT_SIGBUS;

        return VM_FAULT_NOPAGE;
}
EXPORT_SYMBOL_GPL(vmf_insert_page_mkwrite);

vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
                pfn_t pfn)
{
        return __vm_insert_mixed(vma, addr, pfn, false);
}
EXPORT_SYMBOL(vmf_insert_mixed);

/*
 *  If the insertion of PTE failed because someone else already added a
 *  different entry in the mean time, we treat that as success as we assume
 *  the same entry was actually inserted.
 */
vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
                unsigned long addr, pfn_t pfn)
{
        return __vm_insert_mixed(vma, addr, pfn, true);
}

/*
 * maps a range of physical memory into the requested pages. the old
 * mappings are removed. any references to nonexistent pages results
 * in null mappings (currently treated as "copy-on-access")
 */
static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pte_t *pte, *mapped_pte;
        spinlock_t *ptl;
        int err = 0;

        mapped_pte = pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
        if (!pte)
                return -ENOMEM;
        arch_enter_lazy_mmu_mode();
        do {
                BUG_ON(!pte_none(ptep_get(pte)));
                if (!pfn_modify_allowed(pfn, prot)) {
                        err = -EACCES;
                        break;
                }
                set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
                pfn++;
        } while (pte++, addr += PAGE_SIZE, addr != end);
        arch_leave_lazy_mmu_mode();
        pte_unmap_unlock(mapped_pte, ptl);
        return err;
}

static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pmd_t *pmd;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        pmd = pmd_alloc(mm, pud, addr);
        if (!pmd)
                return -ENOMEM;
        VM_BUG_ON(pmd_trans_huge(*pmd));
        do {
                next = pmd_addr_end(addr, end);
                err = remap_pte_range(mm, pmd, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pmd++, addr = next, addr != end);
        return 0;
}

static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        pud_t *pud;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        pud = pud_alloc(mm, p4d, addr);
        if (!pud)
                return -ENOMEM;
        do {
                next = pud_addr_end(addr, end);
                err = remap_pmd_range(mm, pud, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pud++, addr = next, addr != end);
        return 0;
}

static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd,
                        unsigned long addr, unsigned long end,
                        unsigned long pfn, pgprot_t prot)
{
        p4d_t *p4d;
        unsigned long next;
        int err;

        pfn -= addr >> PAGE_SHIFT;
        p4d = p4d_alloc(mm, pgd, addr);
        if (!p4d)
                return -ENOMEM;
        do {
                next = p4d_addr_end(addr, end);
                err = remap_pud_range(mm, p4d, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (p4d++, addr = next, addr != end);
        return 0;
}

static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr,
                unsigned long pfn, unsigned long size, pgprot_t prot)
{
        pgd_t *pgd;
        unsigned long next;
        unsigned long end = addr + PAGE_ALIGN(size);
        struct mm_struct *mm = vma->vm_mm;
        int err;

        if (WARN_ON_ONCE(!PAGE_ALIGNED(addr)))
                return -EINVAL;

        /*
         * Physically remapped pages are special. Tell the
         * rest of the world about it:
         *   VM_IO tells people not to look at these pages
         *        (accesses can have side effects).
         *   VM_PFNMAP tells the core MM that the base pages are just
         *        raw PFN mappings, and do not have a "struct page" associated
         *        with them.
         *   VM_DONTEXPAND
         *      Disable vma merging and expanding with mremap().
         *   VM_DONTDUMP
         *      Omit vma from core dump, even when VM_IO turned off.
         *
         * There's a horrible special case to handle copy-on-write
         * behaviour that some programs depend on. We mark the "original"
         * un-COW'ed pages by matching them up with "vma->vm_pgoff".
         * See vm_normal_page() for details.
         */
        if (is_cow_mapping(vma->vm_flags)) {
                if (addr != vma->vm_start || end != vma->vm_end)
                        return -EINVAL;
                vma->vm_pgoff = pfn;
        }

        vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);

        BUG_ON(addr >= end);
        pfn -= addr >> PAGE_SHIFT;
        pgd = pgd_offset(mm, addr);
        flush_cache_range(vma, addr, end);
        do {
                next = pgd_addr_end(addr, end);
                err = remap_p4d_range(mm, pgd, addr, next,
                                pfn + (addr >> PAGE_SHIFT), prot);
                if (err)
                        return err;
        } while (pgd++, addr = next, addr != end);

        return 0;
}

/*
 * Variant of remap_pfn_range that does not call track_pfn_remap.  The caller
 * must have pre-validated the caching bits of the pgprot_t.
 */
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
                unsigned long pfn, unsigned long size, pgprot_t prot)
{
        int error = remap_pfn_range_internal(vma, addr, pfn, size, prot);

        if (!error)
                return 0;

        /*
         * A partial pfn range mapping is dangerous: it does not
         * maintain page reference counts, and callers may free
         * pages due to the error. So zap it early.
         */
        zap_page_range_single(vma, addr, size, NULL);
        return error;
}

#ifdef __HAVE_PFNMAP_TRACKING
static inline struct pfnmap_track_ctx *pfnmap_track_ctx_alloc(unsigned long pfn,
                unsigned long size, pgprot_t *prot)
{
        struct pfnmap_track_ctx *ctx;

        if (pfnmap_track(pfn, size, prot))
                return ERR_PTR(-EINVAL);

        ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
        if (unlikely(!ctx)) {
                pfnmap_untrack(pfn, size);
                return ERR_PTR(-ENOMEM);
        }

        ctx->pfn = pfn;
        ctx->size = size;
        kref_init(&ctx->kref);
        return ctx;
}

void pfnmap_track_ctx_release(struct kref *ref)
{
        struct pfnmap_track_ctx *ctx = container_of(ref, struct pfnmap_track_ctx, kref);

        pfnmap_untrack(ctx->pfn, ctx->size);
        kfree(ctx);
}
#endif /* __HAVE_PFNMAP_TRACKING */

/**
 * remap_pfn_range - remap kernel memory to userspace
 * @vma: user vma to map to
 * @addr: target page aligned user address to start at
 * @pfn: page frame number of kernel physical memory address
 * @size: size of mapping area
 * @prot: page protection flags for this mapping
 *
 * Note: this is only safe if the mm semaphore is held when called.
 *
 * Return: %0 on success, negative error code otherwise.
 */
#ifdef __HAVE_PFNMAP_TRACKING
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
                    unsigned long pfn, unsigned long size, pgprot_t prot)
{
        struct pfnmap_track_ctx *ctx = NULL;
        int err;

        size = PAGE_ALIGN(size);

        /*
         * If we cover the full VMA, we'll perform actual tracking, and
         * remember to untrack when the last reference to our tracking
         * context from a VMA goes away. We'll keep tracking the whole pfn
         * range even during VMA splits and partial unmapping.
         *
         * If we only cover parts of the VMA, we'll only setup the cachemode
         * in the pgprot for the pfn range.
         */
        if (addr == vma->vm_start && addr + size == vma->vm_end) {
                if (vma->pfnmap_track_ctx)
                        return -EINVAL;
                ctx = pfnmap_track_ctx_alloc(pfn, size, &prot);
                if (IS_ERR(ctx))
                        return PTR_ERR(ctx);
        } else if (pfnmap_setup_cachemode(pfn, size, &prot)) {
                return -EINVAL;
        }

        err = remap_pfn_range_notrack(vma, addr, pfn, size, prot);
        if (ctx) {
                if (err)
                        kref_put(&ctx->kref, pfnmap_track_ctx_release);
                else
                        vma->pfnmap_track_ctx = ctx;
        }
        return err;
}

#else
int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
                    unsigned long pfn, unsigned long size, pgprot_t prot)
{
        return remap_pfn_range_notrack(vma, addr, pfn, size, prot);
}
#endif
EXPORT_SYMBOL(remap_pfn_range);

/**
 * vm_iomap_memory - remap memory to userspace
 * @vma: user vma to map to
 * @start: start of the physical memory to be mapped
 * @len: size of area
 *
 * This is a simplified io_remap_pfn_range() for common driver use. The
 * driver just needs to give us the physical memory range to be mapped,
 * we'll figure out the rest from the vma information.
 *
 * NOTE! Some drivers might want to tweak vma->vm_page_prot first to get
 * whatever write-combining details or similar.
 *
 * Return: %0 on success, negative error code otherwise.
 */
int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
{
        unsigned long vm_len, pfn, pages;

        /* Check that the physical memory area passed in looks valid */
        if (start + len < start)
                return -EINVAL;
        /*
         * You *really* shouldn't map things that aren't page-aligned,
         * but we've historically allowed it because IO memory might
         * just have smaller alignment.
         */
        len += start & ~PAGE_MASK;
        pfn = start >> PAGE_SHIFT;
        pages = (len + ~PAGE_MASK) >> PAGE_SHIFT;
        if (pfn + pages < pfn)
                return -EINVAL;

        /* We start the mapping 'vm_pgoff' pages into the area */
        if (vma->vm_pgoff > pages)
                return -EINVAL;
        pfn += vma->vm_pgoff;
        pages -= vma->vm_pgoff;

        /* Can we fit all of the mapping? */
        vm_len = vma->vm_end - vma->vm_start;
        if (vm_len >> PAGE_SHIFT > pages)
                return -EINVAL;

        /* Ok, let it rip */
        return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
}
EXPORT_SYMBOL(vm_iomap_memory);

static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pte_t *pte, *mapped_pte;
        int err = 0;
        spinlock_t *ptl;

        if (create) {
                mapped_pte = pte = (mm == &init_mm) ?
                        pte_alloc_kernel_track(pmd, addr, mask) :
                        pte_alloc_map_lock(mm, pmd, addr, &ptl);
                if (!pte)
                        return -ENOMEM;
        } else {
                mapped_pte = pte = (mm == &init_mm) ?
                        pte_offset_kernel(pmd, addr) :
                        pte_offset_map_lock(mm, pmd, addr, &ptl);
                if (!pte)
                        return -EINVAL;
        }

        arch_enter_lazy_mmu_mode();

        if (fn) {
                do {
                        if (create || !pte_none(ptep_get(pte))) {
                                err = fn(pte, addr, data);
                                if (err)
                                        break;
                        }
                } while (pte++, addr += PAGE_SIZE, addr != end);
        }
        *mask |= PGTBL_PTE_MODIFIED;

        arch_leave_lazy_mmu_mode();

        if (mm != &init_mm)
                pte_unmap_unlock(mapped_pte, ptl);
        return err;
}

static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pmd_t *pmd;
        unsigned long next;
        int err = 0;

        BUG_ON(pud_leaf(*pud));

        if (create) {
                pmd = pmd_alloc_track(mm, pud, addr, mask);
                if (!pmd)
                        return -ENOMEM;
        } else {
                pmd = pmd_offset(pud, addr);
        }
        do {
                next = pmd_addr_end(addr, end);
                if (pmd_none(*pmd) && !create)
                        continue;
                if (WARN_ON_ONCE(pmd_leaf(*pmd)))
                        return -EINVAL;
                if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) {
                        if (!create)
                                continue;
                        pmd_clear_bad(pmd);
                }
                err = apply_to_pte_range(mm, pmd, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (pmd++, addr = next, addr != end);

        return err;
}

static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        pud_t *pud;
        unsigned long next;
        int err = 0;

        if (create) {
                pud = pud_alloc_track(mm, p4d, addr, mask);
                if (!pud)
                        return -ENOMEM;
        } else {
                pud = pud_offset(p4d, addr);
        }
        do {
                next = pud_addr_end(addr, end);
                if (pud_none(*pud) && !create)
                        continue;
                if (WARN_ON_ONCE(pud_leaf(*pud)))
                        return -EINVAL;
                if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) {
                        if (!create)
                                continue;
                        pud_clear_bad(pud);
                }
                err = apply_to_pmd_range(mm, pud, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (pud++, addr = next, addr != end);

        return err;
}

static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd,
                                     unsigned long addr, unsigned long end,
                                     pte_fn_t fn, void *data, bool create,
                                     pgtbl_mod_mask *mask)
{
        p4d_t *p4d;
        unsigned long next;
        int err = 0;

        if (create) {
                p4d = p4d_alloc_track(mm, pgd, addr, mask);
                if (!p4d)
                        return -ENOMEM;
        } else {
                p4d = p4d_offset(pgd, addr);
        }
        do {
                next = p4d_addr_end(addr, end);
                if (p4d_none(*p4d) && !create)
                        continue;
                if (WARN_ON_ONCE(p4d_leaf(*p4d)))
                        return -EINVAL;
                if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) {
                        if (!create)
                                continue;
                        p4d_clear_bad(p4d);
                }
                err = apply_to_pud_range(mm, p4d, addr, next,
                                         fn, data, create, mask);
                if (err)
                        break;
        } while (p4d++, addr = next, addr != end);

        return err;
}

static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                                 unsigned long size, pte_fn_t fn,
                                 void *data, bool create)
{
        pgd_t *pgd;
        unsigned long start = addr, next;
        unsigned long end = addr + size;
        pgtbl_mod_mask mask = 0;
        int err = 0;

        if (WARN_ON(addr >= end))
                return -EINVAL;

        pgd = pgd_offset(mm, addr);
        do {
                next = pgd_addr_end(addr, end);
                if (pgd_none(*pgd) && !create)
                        continue;
                if (WARN_ON_ONCE(pgd_leaf(*pgd))) {
                        err = -EINVAL;
                        break;
                }
                if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) {
                        if (!create)
                                continue;
                        pgd_clear_bad(pgd);
                }
                err = apply_to_p4d_range(mm, pgd, addr, next,
                                         fn, data, create, &mask);
                if (err)
                        break;
        } while (pgd++, addr = next, addr != end);

        if (mask & ARCH_PAGE_TABLE_SYNC_MASK)
                arch_sync_kernel_mappings(start, start + size);

        return err;
}

/*
 * Scan a region of virtual memory, filling in page tables as necessary
 * and calling a provided function on each leaf page table.
 */
int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
                        unsigned long size, pte_fn_t fn, void *data)
{
        return __apply_to_page_range(mm, addr, size, fn, data, true);
}
EXPORT_SYMBOL_GPL(apply_to_page_range);

/*
 * Scan a region of virtual memory, calling a provided function on
 * each leaf page table where it exists.
 *
 * Unlike apply_to_page_range, this does _not_ fill in page tables
 * where they are absent.
 */
int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
                                 unsigned long size, pte_fn_t fn, void *data)
{
        return __apply_to_page_range(mm, addr, size, fn, data, false);
}

/*
 * handle_pte_fault chooses page fault handler according to an entry which was
 * read non-atomically.  Before making any commitment, on those architectures
 * or configurations (e.g. i386 with PAE) which might give a mix of unmatched
 * parts, do_swap_page must check under lock before unmapping the pte and
 * proceeding (but do_wp_page is only called after already making such a check;
 * and do_anonymous_page can safely check later on).
 */
static inline int pte_unmap_same(struct vm_fault *vmf)
{
        int same = 1;
#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPTION)
        if (sizeof(pte_t) > sizeof(unsigned long)) {
                spin_lock(vmf->ptl);
                same = pte_same(ptep_get(vmf->pte), vmf->orig_pte);
                spin_unlock(vmf->ptl);
        }
#endif
        pte_unmap(vmf->pte);
        vmf->pte = NULL;
        return same;
}

/*
 * Return:
 *        0:                copied succeeded
 *        -EHWPOISON:        copy failed due to hwpoison in source page
 *        -EAGAIN:        copied failed (some other reason)
 */
static inline int __wp_page_copy_user(struct page *dst, struct page *src,
                                      struct vm_fault *vmf)
{
        int ret;
        void *kaddr;
        void __user *uaddr;
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        unsigned long addr = vmf->address;

        if (likely(src)) {
                if (copy_mc_user_highpage(dst, src, addr, vma))
                        return -EHWPOISON;
                return 0;
        }

        /*
         * If the source page was a PFN mapping, we don't have
         * a "struct page" for it. We do a best-effort copy by
         * just copying from the original user address. If that
         * fails, we just zero-fill it. Live with it.
         */
        kaddr = kmap_local_page(dst);
        pagefault_disable();
        uaddr = (void __user *)(addr & PAGE_MASK);

        /*
         * On architectures with software "accessed" bits, we would
         * take a double page fault, so mark it accessed here.
         */
        vmf->pte = NULL;
        if (!arch_has_hw_pte_young() && !pte_young(vmf->orig_pte)) {
                pte_t entry;

                vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
                if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                        /*
                         * Other thread has already handled the fault
                         * and update local tlb only
                         */
                        if (vmf->pte)
                                update_mmu_tlb(vma, addr, vmf->pte);
                        ret = -EAGAIN;
                        goto pte_unlock;
                }

                entry = pte_mkyoung(vmf->orig_pte);
                if (ptep_set_access_flags(vma, addr, vmf->pte, entry, 0))
                        update_mmu_cache_range(vmf, vma, addr, vmf->pte, 1);
        }

        /*
         * This really shouldn't fail, because the page is there
         * in the page tables. But it might just be unreadable,
         * in which case we just give up and fill the result with
         * zeroes.
         */
        if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
                if (vmf->pte)
                        goto warn;

                /* Re-validate under PTL if the page is still mapped */
                vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl);
                if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                        /* The PTE changed under us, update local tlb */
                        if (vmf->pte)
                                update_mmu_tlb(vma, addr, vmf->pte);
                        ret = -EAGAIN;
                        goto pte_unlock;
                }

                /*
                 * The same page can be mapped back since last copy attempt.
                 * Try to copy again under PTL.
                 */
                if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) {
                        /*
                         * Give a warn in case there can be some obscure
                         * use-case
                         */
warn:
                        WARN_ON_ONCE(1);
                        clear_page(kaddr);
                }
        }

        ret = 0;

pte_unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        pagefault_enable();
        kunmap_local(kaddr);
        flush_dcache_page(dst);

        return ret;
}

static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma)
{
        struct file *vm_file = vma->vm_file;

        if (vm_file)
                return mapping_gfp_mask(vm_file->f_mapping) | __GFP_FS | __GFP_IO;

        /*
         * Special mappings (e.g. VDSO) do not have any file so fake
         * a default GFP_KERNEL for them.
         */
        return GFP_KERNEL;
}

/*
 * Notify the address space that the page is about to become writable so that
 * it can prohibit this or wait for the page to get into an appropriate state.
 *
 * We do this without the lock held, so that it can sleep if it needs to.
 */
static vm_fault_t do_page_mkwrite(struct vm_fault *vmf, struct folio *folio)
{
        vm_fault_t ret;
        unsigned int old_flags = vmf->flags;

        vmf->flags = FAULT_FLAG_WRITE|FAULT_FLAG_MKWRITE;

        if (vmf->vma->vm_file &&
            IS_SWAPFILE(vmf->vma->vm_file->f_mapping->host))
                return VM_FAULT_SIGBUS;

        ret = vmf->vma->vm_ops->page_mkwrite(vmf);
        /* Restore original flags so that caller is not surprised */
        vmf->flags = old_flags;
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
                return ret;
        if (unlikely(!(ret & VM_FAULT_LOCKED))) {
                folio_lock(folio);
                if (!folio->mapping) {
                        folio_unlock(folio);
                        return 0; /* retry */
                }
                ret |= VM_FAULT_LOCKED;
        } else
                VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
        return ret;
}

/*
 * Handle dirtying of a page in shared file mapping on a write fault.
 *
 * The function expects the page to be locked and unlocks it.
 */
static vm_fault_t fault_dirty_shared_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct address_space *mapping;
        struct folio *folio = page_folio(vmf->page);
        bool dirtied;
        bool page_mkwrite = vma->vm_ops && vma->vm_ops->page_mkwrite;

        dirtied = folio_mark_dirty(folio);
        VM_BUG_ON_FOLIO(folio_test_anon(folio), folio);
        /*
         * Take a local copy of the address_space - folio.mapping may be zeroed
         * by truncate after folio_unlock().   The address_space itself remains
         * pinned by vma->vm_file's reference.  We rely on folio_unlock()'s
         * release semantics to prevent the compiler from undoing this copying.
         */
        mapping = folio_raw_mapping(folio);
        folio_unlock(folio);

        if (!page_mkwrite)
                file_update_time(vma->vm_file);

        /*
         * Throttle page dirtying rate down to writeback speed.
         *
         * mapping may be NULL here because some device drivers do not
         * set page.mapping but still dirty their pages
         *
         * Drop the mmap_lock before waiting on IO, if we can. The file
         * is pinning the mapping, as per above.
         */
        if ((dirtied || page_mkwrite) && mapping) {
                struct file *fpin;

                fpin = maybe_unlock_mmap_for_io(vmf, NULL);
                balance_dirty_pages_ratelimited(mapping);
                if (fpin) {
                        fput(fpin);
                        return VM_FAULT_COMPLETED;
                }
        }

        return 0;
}

/*
 * Handle write page faults for pages that can be reused in the current vma
 *
 * This can happen either due to the mapping being with the VM_SHARED flag,
 * or due to us being the last reference standing to the page. In either
 * case, all we need to do here is to mark the page as writable and update
 * any related book-keeping.
 */
static inline void wp_page_reuse(struct vm_fault *vmf, struct folio *folio)
        __releases(vmf->ptl)
{
        struct vm_area_struct *vma = vmf->vma;
        pte_t entry;

        VM_BUG_ON(!(vmf->flags & FAULT_FLAG_WRITE));
        VM_WARN_ON(is_zero_pfn(pte_pfn(vmf->orig_pte)));

        if (folio) {
                VM_BUG_ON(folio_test_anon(folio) &&
                          !PageAnonExclusive(vmf->page));
                /*
                 * Clear the folio's cpupid information as the existing
                 * information potentially belongs to a now completely
                 * unrelated process.
                 */
                folio_xchg_last_cpupid(folio, (1 << LAST_CPUPID_SHIFT) - 1);
        }

        flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
        entry = pte_mkyoung(vmf->orig_pte);
        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
                update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        count_vm_event(PGREUSE);
}

/*
 * We could add a bitflag somewhere, but for now, we know that all
 * vm_ops that have a ->map_pages have been audited and don't need
 * the mmap_lock to be held.
 */
static inline vm_fault_t vmf_can_call_fault(const struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        if (vma->vm_ops->map_pages || !(vmf->flags & FAULT_FLAG_VMA_LOCK))
                return 0;
        vma_end_read(vma);
        return VM_FAULT_RETRY;
}

/**
 * __vmf_anon_prepare - Prepare to handle an anonymous fault.
 * @vmf: The vm_fault descriptor passed from the fault handler.
 *
 * When preparing to insert an anonymous page into a VMA from a
 * fault handler, call this function rather than anon_vma_prepare().
 * If this vma does not already have an associated anon_vma and we are
 * only protected by the per-VMA lock, the caller must retry with the
 * mmap_lock held.  __anon_vma_prepare() will look at adjacent VMAs to
 * determine if this VMA can share its anon_vma, and that's not safe to
 * do with only the per-VMA lock held for this VMA.
 *
 * Return: 0 if fault handling can proceed.  Any other value should be
 * returned to the caller.
 */
vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret = 0;

        if (likely(vma->anon_vma))
                return 0;
        if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
                if (!mmap_read_trylock(vma->vm_mm))
                        return VM_FAULT_RETRY;
        }
        if (__anon_vma_prepare(vma))
                ret = VM_FAULT_OOM;
        if (vmf->flags & FAULT_FLAG_VMA_LOCK)
                mmap_read_unlock(vma->vm_mm);
        return ret;
}

/*
 * Handle the case of a page which we actually need to copy to a new page,
 * either due to COW or unsharing.
 *
 * Called with mmap_lock locked and the old page referenced, but
 * without the ptl held.
 *
 * High level logic flow:
 *
 * - Allocate a page, copy the content of the old page to the new one.
 * - Handle book keeping and accounting - cgroups, mmu-notifiers, etc.
 * - Take the PTL. If the pte changed, bail out and release the allocated page
 * - If the pte is still the way we remember it, update the page table and all
 *   relevant references. This includes dropping the reference the page-table
 *   held to the old page, as well as updating the rmap.
 * - In any case, unlock the PTL and drop the reference we took to the old page.
 */
static vm_fault_t wp_page_copy(struct vm_fault *vmf)
{
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *mm = vma->vm_mm;
        struct folio *old_folio = NULL;
        struct folio *new_folio = NULL;
        pte_t entry;
        int page_copied = 0;
        struct mmu_notifier_range range;
        vm_fault_t ret;
        bool pfn_is_zero;

        delayacct_wpcopy_start();

        if (vmf->page)
                old_folio = page_folio(vmf->page);
        ret = vmf_anon_prepare(vmf);
        if (unlikely(ret))
                goto out;

        pfn_is_zero = is_zero_pfn(pte_pfn(vmf->orig_pte));
        new_folio = folio_prealloc(mm, vma, vmf->address, pfn_is_zero);
        if (!new_folio)
                goto oom;

        if (!pfn_is_zero) {
                int err;

                err = __wp_page_copy_user(&new_folio->page, vmf->page, vmf);
                if (err) {
                        /*
                         * COW failed, if the fault was solved by other,
                         * it's fine. If not, userspace would re-fault on
                         * the same address and we will handle the fault
                         * from the second attempt.
                         * The -EHWPOISON case will not be retried.
                         */
                        folio_put(new_folio);
                        if (old_folio)
                                folio_put(old_folio);

                        delayacct_wpcopy_end();
                        return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
                }
                kmsan_copy_page_meta(&new_folio->page, vmf->page);
        }

        __folio_mark_uptodate(new_folio);

        mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
                                vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE);
        mmu_notifier_invalidate_range_start(&range);

        /*
         * Re-check the pte - we dropped the lock
         */
        vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
        if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                if (old_folio) {
                        if (!folio_test_anon(old_folio)) {
                                dec_mm_counter(mm, mm_counter_file(old_folio));
                                inc_mm_counter(mm, MM_ANONPAGES);
                        }
                } else {
                        ksm_might_unmap_zero_page(mm, vmf->orig_pte);
                        inc_mm_counter(mm, MM_ANONPAGES);
                }
                flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
                entry = folio_mk_pte(new_folio, vma->vm_page_prot);
                entry = pte_sw_mkyoung(entry);
                if (unlikely(unshare)) {
                        if (pte_soft_dirty(vmf->orig_pte))
                                entry = pte_mksoft_dirty(entry);
                        if (pte_uffd_wp(vmf->orig_pte))
                                entry = pte_mkuffd_wp(entry);
                } else {
                        entry = maybe_mkwrite(pte_mkdirty(entry), vma);
                }

                /*
                 * Clear the pte entry and flush it first, before updating the
                 * pte with the new entry, to keep TLBs on different CPUs in
                 * sync. This code used to set the new PTE then flush TLBs, but
                 * that left a window where the new PTE could be loaded into
                 * some TLBs while the old PTE remains in others.
                 */
                ptep_clear_flush(vma, vmf->address, vmf->pte);
                folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(new_folio, vma);
                BUG_ON(unshare && pte_write(entry));
                set_pte_at(mm, vmf->address, vmf->pte, entry);
                update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
                if (old_folio) {
                        /*
                         * Only after switching the pte to the new page may
                         * we remove the mapcount here. Otherwise another
                         * process may come and find the rmap count decremented
                         * before the pte is switched to the new page, and
                         * "reuse" the old page writing into it while our pte
                         * here still points into it and can be read by other
                         * threads.
                         *
                         * The critical issue is to order this
                         * folio_remove_rmap_pte() with the ptp_clear_flush
                         * above. Those stores are ordered by (if nothing else,)
                         * the barrier present in the atomic_add_negative
                         * in folio_remove_rmap_pte();
                         *
                         * Then the TLB flush in ptep_clear_flush ensures that
                         * no process can access the old page before the
                         * decremented mapcount is visible. And the old page
                         * cannot be reused until after the decremented
                         * mapcount is visible. So transitively, TLBs to
                         * old page will be flushed before it can be reused.
                         */
                        folio_remove_rmap_pte(old_folio, vmf->page, vma);
                }

                /* Free the old page.. */
                new_folio = old_folio;
                page_copied = 1;
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        } else if (vmf->pte) {
                update_mmu_tlb(vma, vmf->address, vmf->pte);
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        }

        mmu_notifier_invalidate_range_end(&range);

        if (new_folio)
                folio_put(new_folio);
        if (old_folio) {
                if (page_copied)
                        free_swap_cache(old_folio);
                folio_put(old_folio);
        }

        delayacct_wpcopy_end();
        return 0;
oom:
        ret = VM_FAULT_OOM;
out:
        if (old_folio)
                folio_put(old_folio);

        delayacct_wpcopy_end();
        return ret;
}

/**
 * finish_mkwrite_fault - finish page fault for a shared mapping, making PTE
 *                          writeable once the page is prepared
 *
 * @vmf: structure describing the fault
 * @folio: the folio of vmf->page
 *
 * This function handles all that is needed to finish a write page fault in a
 * shared mapping due to PTE being read-only once the mapped page is prepared.
 * It handles locking of PTE and modifying it.
 *
 * The function expects the page to be locked or other protection against
 * concurrent faults / writeback (such as DAX radix tree locks).
 *
 * Return: %0 on success, %VM_FAULT_NOPAGE when PTE got changed before
 * we acquired PTE lock.
 */
static vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf, struct folio *folio)
{
        WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
        vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
                                       &vmf->ptl);
        if (!vmf->pte)
                return VM_FAULT_NOPAGE;
        /*
         * We might have raced with another page fault while we released the
         * pte_offset_map_lock.
         */
        if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) {
                update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return VM_FAULT_NOPAGE;
        }
        wp_page_reuse(vmf, folio);
        return 0;
}

/*
 * Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
 * mapping
 */
static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
                vm_fault_t ret;

                pte_unmap_unlock(vmf->pte, vmf->ptl);
                ret = vmf_can_call_fault(vmf);
                if (ret)
                        return ret;

                vmf->flags |= FAULT_FLAG_MKWRITE;
                ret = vma->vm_ops->pfn_mkwrite(vmf);
                if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))
                        return ret;
                return finish_mkwrite_fault(vmf, NULL);
        }
        wp_page_reuse(vmf, NULL);
        return 0;
}

static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio)
        __releases(vmf->ptl)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret = 0;

        folio_get(folio);

        if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
                vm_fault_t tmp;

                pte_unmap_unlock(vmf->pte, vmf->ptl);
                tmp = vmf_can_call_fault(vmf);
                if (tmp) {
                        folio_put(folio);
                        return tmp;
                }

                tmp = do_page_mkwrite(vmf, folio);
                if (unlikely(!tmp || (tmp &
                                      (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
                        folio_put(folio);
                        return tmp;
                }
                tmp = finish_mkwrite_fault(vmf, folio);
                if (unlikely(tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) {
                        folio_unlock(folio);
                        folio_put(folio);
                        return tmp;
                }
        } else {
                wp_page_reuse(vmf, folio);
                folio_lock(folio);
        }
        ret |= fault_dirty_shared_page(vmf);
        folio_put(folio);

        return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
                struct vm_area_struct *vma)
{
        bool exclusive = false;

        /* Let's just free up a large folio if only a single page is mapped. */
        if (folio_large_mapcount(folio) <= 1)
                return false;

        /*
         * The assumption for anonymous folios is that each page can only get
         * mapped once into each MM. The only exception are KSM folios, which
         * are always small.
         *
         * Each taken mapcount must be paired with exactly one taken reference,
         * whereby the refcount must be incremented before the mapcount when
         * mapping a page, and the refcount must be decremented after the
         * mapcount when unmapping a page.
         *
         * If all folio references are from mappings, and all mappings are in
         * the page tables of this MM, then this folio is exclusive to this MM.
         */
        if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids))
                return false;

        VM_WARN_ON_ONCE(folio_test_ksm(folio));

        if (unlikely(folio_test_swapcache(folio))) {
                /*
                 * Note: freeing up the swapcache will fail if some PTEs are
                 * still swap entries.
                 */
                if (!folio_trylock(folio))
                        return false;
                folio_free_swap(folio);
                folio_unlock(folio);
        }

        if (folio_large_mapcount(folio) != folio_ref_count(folio))
                return false;

        /* Stabilize the mapcount vs. refcount and recheck. */
        folio_lock_large_mapcount(folio);
        VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_ref_count(folio), folio);

        if (test_bit(FOLIO_MM_IDS_SHARED_BITNUM, &folio->_mm_ids))
                goto unlock;
        if (folio_large_mapcount(folio) != folio_ref_count(folio))
                goto unlock;

        VM_WARN_ON_ONCE_FOLIO(folio_large_mapcount(folio) > folio_nr_pages(folio), folio);
        VM_WARN_ON_ONCE_FOLIO(folio_entire_mapcount(folio), folio);
        VM_WARN_ON_ONCE(folio_mm_id(folio, 0) != vma->vm_mm->mm_id &&
                        folio_mm_id(folio, 1) != vma->vm_mm->mm_id);

        /*
         * Do we need the folio lock? Likely not. If there would have been
         * references from page migration/swapout, we would have detected
         * an additional folio reference and never ended up here.
         */
        exclusive = true;
unlock:
        folio_unlock_large_mapcount(folio);
        return exclusive;
}
#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
static bool __wp_can_reuse_large_anon_folio(struct folio *folio,
                struct vm_area_struct *vma)
{
        BUILD_BUG();
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static bool wp_can_reuse_anon_folio(struct folio *folio,
                                    struct vm_area_struct *vma)
{
        if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio))
                return __wp_can_reuse_large_anon_folio(folio, vma);

        /*
         * We have to verify under folio lock: these early checks are
         * just an optimization to avoid locking the folio and freeing
         * the swapcache if there is little hope that we can reuse.
         *
         * KSM doesn't necessarily raise the folio refcount.
         */
        if (folio_test_ksm(folio) || folio_ref_count(folio) > 3)
                return false;
        if (!folio_test_lru(folio))
                /*
                 * We cannot easily detect+handle references from
                 * remote LRU caches or references to LRU folios.
                 */
                lru_add_drain();
        if (folio_ref_count(folio) > 1 + folio_test_swapcache(folio))
                return false;
        if (!folio_trylock(folio))
                return false;
        if (folio_test_swapcache(folio))
                folio_free_swap(folio);
        if (folio_test_ksm(folio) || folio_ref_count(folio) != 1) {
                folio_unlock(folio);
                return false;
        }
        /*
         * Ok, we've got the only folio reference from our mapping
         * and the folio is locked, it's dark out, and we're wearing
         * sunglasses. Hit it.
         */
        folio_move_anon_rmap(folio, vma);
        folio_unlock(folio);
        return true;
}

/*
 * This routine handles present pages, when
 * * users try to write to a shared page (FAULT_FLAG_WRITE)
 * * GUP wants to take a R/O pin on a possibly shared anonymous page
 *   (FAULT_FLAG_UNSHARE)
 *
 * It is done by copying the page to a new address and decrementing the
 * shared-page counter for the old page.
 *
 * Note that this routine assumes that the protection checks have been
 * done by the caller (the low-level page fault routine in most cases).
 * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've
 * done any necessary COW.
 *
 * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even
 * though the page will change only once the write actually happens. This
 * avoids a few races, and potentially makes it more efficient.
 *
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), with pte both mapped and locked.
 * We return with mmap_lock still held, but pte unmapped and unlocked.
 */
static vm_fault_t do_wp_page(struct vm_fault *vmf)
        __releases(vmf->ptl)
{
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio = NULL;
        pte_t pte;

        if (likely(!unshare)) {
                if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) {
                        if (!userfaultfd_wp_async(vma)) {
                                pte_unmap_unlock(vmf->pte, vmf->ptl);
                                return handle_userfault(vmf, VM_UFFD_WP);
                        }

                        /*
                         * Nothing needed (cache flush, TLB invalidations,
                         * etc.) because we're only removing the uffd-wp bit,
                         * which is completely invisible to the user.
                         */
                        pte = pte_clear_uffd_wp(ptep_get(vmf->pte));

                        set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
                        /*
                         * Update this to be prepared for following up CoW
                         * handling
                         */
                        vmf->orig_pte = pte;
                }

                /*
                 * Userfaultfd write-protect can defer flushes. Ensure the TLB
                 * is flushed in this case before copying.
                 */
                if (unlikely(userfaultfd_wp(vmf->vma) &&
                             mm_tlb_flush_pending(vmf->vma->vm_mm)))
                        flush_tlb_page(vmf->vma, vmf->address);
        }

        vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);

        if (vmf->page)
                folio = page_folio(vmf->page);

        /*
         * Shared mapping: we are guaranteed to have VM_WRITE and
         * FAULT_FLAG_WRITE set at this point.
         */
        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                /*
                 * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
                 * VM_PFNMAP VMA. FS DAX also wants ops->pfn_mkwrite called.
                 *
                 * We should not cow pages in a shared writeable mapping.
                 * Just mark the pages writable and/or call ops->pfn_mkwrite.
                 */
                if (!vmf->page || is_fsdax_page(vmf->page)) {
                        vmf->page = NULL;
                        return wp_pfn_shared(vmf);
                }
                return wp_page_shared(vmf, folio);
        }

        /*
         * Private mapping: create an exclusive anonymous page copy if reuse
         * is impossible. We might miss VM_WRITE for FOLL_FORCE handling.
         *
         * If we encounter a page that is marked exclusive, we must reuse
         * the page without further checks.
         */
        if (folio && folio_test_anon(folio) &&
            (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) {
                if (!PageAnonExclusive(vmf->page))
                        SetPageAnonExclusive(vmf->page);
                if (unlikely(unshare)) {
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                        return 0;
                }
                wp_page_reuse(vmf, folio);
                return 0;
        }
        /*
         * Ok, we need to copy. Oh, well..
         */
        if (folio)
                folio_get(folio);

        pte_unmap_unlock(vmf->pte, vmf->ptl);
#ifdef CONFIG_KSM
        if (folio && folio_test_ksm(folio))
                count_vm_event(COW_KSM);
#endif
        return wp_page_copy(vmf);
}

static void unmap_mapping_range_vma(struct vm_area_struct *vma,
                unsigned long start_addr, unsigned long end_addr,
                struct zap_details *details)
{
        zap_page_range_single(vma, start_addr, end_addr - start_addr, details);
}

static inline void unmap_mapping_range_tree(struct rb_root_cached *root,
                                            pgoff_t first_index,
                                            pgoff_t last_index,
                                            struct zap_details *details)
{
        struct vm_area_struct *vma;
        pgoff_t vba, vea, zba, zea;

        vma_interval_tree_foreach(vma, root, first_index, last_index) {
                vba = vma->vm_pgoff;
                vea = vba + vma_pages(vma) - 1;
                zba = max(first_index, vba);
                zea = min(last_index, vea);

                unmap_mapping_range_vma(vma,
                        ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
                        ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
                                details);
        }
}

/**
 * unmap_mapping_folio() - Unmap single folio from processes.
 * @folio: The locked folio to be unmapped.
 *
 * Unmap this folio from any userspace process which still has it mmaped.
 * Typically, for efficiency, the range of nearby pages has already been
 * unmapped by unmap_mapping_pages() or unmap_mapping_range().  But once
 * truncation or invalidation holds the lock on a folio, it may find that
 * the page has been remapped again: and then uses unmap_mapping_folio()
 * to unmap it finally.
 */
void unmap_mapping_folio(struct folio *folio)
{
        struct address_space *mapping = folio->mapping;
        struct zap_details details = { };
        pgoff_t        first_index;
        pgoff_t        last_index;

        VM_BUG_ON(!folio_test_locked(folio));

        first_index = folio->index;
        last_index = folio_next_index(folio) - 1;

        details.even_cows = false;
        details.single_folio = folio;
        details.zap_flags = ZAP_FLAG_DROP_MARKER;

        i_mmap_lock_read(mapping);
        if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
                unmap_mapping_range_tree(&mapping->i_mmap, first_index,
                                         last_index, &details);
        i_mmap_unlock_read(mapping);
}

/**
 * unmap_mapping_pages() - Unmap pages from processes.
 * @mapping: The address space containing pages to be unmapped.
 * @start: Index of first page to be unmapped.
 * @nr: Number of pages to be unmapped.  0 to unmap to end of file.
 * @even_cows: Whether to unmap even private COWed pages.
 *
 * Unmap the pages in this address space from any userspace process which
 * has them mmaped.  Generally, you want to remove COWed pages as well when
 * a file is being truncated, but not when invalidating pages from the page
 * cache.
 */
void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
                pgoff_t nr, bool even_cows)
{
        struct zap_details details = { };
        pgoff_t        first_index = start;
        pgoff_t        last_index = start + nr - 1;

        details.even_cows = even_cows;
        if (last_index < first_index)
                last_index = ULONG_MAX;

        i_mmap_lock_read(mapping);
        if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
                unmap_mapping_range_tree(&mapping->i_mmap, first_index,
                                         last_index, &details);
        i_mmap_unlock_read(mapping);
}
EXPORT_SYMBOL_GPL(unmap_mapping_pages);

/**
 * unmap_mapping_range - unmap the portion of all mmaps in the specified
 * address_space corresponding to the specified byte range in the underlying
 * file.
 *
 * @mapping: the address space containing mmaps to be unmapped.
 * @holebegin: byte in first page to unmap, relative to the start of
 * the underlying file.  This will be rounded down to a PAGE_SIZE
 * boundary.  Note that this is different from truncate_pagecache(), which
 * must keep the partial page.  In contrast, we must get rid of
 * partial pages.
 * @holelen: size of prospective hole in bytes.  This will be rounded
 * up to a PAGE_SIZE boundary.  A holelen of zero truncates to the
 * end of the file.
 * @even_cows: 1 when truncating a file, unmap even private COWed pages;
 * but 0 when invalidating pagecache, don't throw away private data.
 */
void unmap_mapping_range(struct address_space *mapping,
                loff_t const holebegin, loff_t const holelen, int even_cows)
{
        pgoff_t hba = (pgoff_t)(holebegin) >> PAGE_SHIFT;
        pgoff_t hlen = ((pgoff_t)(holelen) + PAGE_SIZE - 1) >> PAGE_SHIFT;

        /* Check for overflow. */
        if (sizeof(holelen) > sizeof(hlen)) {
                long long holeend =
                        (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
                if (holeend & ~(long long)ULONG_MAX)
                        hlen = ULONG_MAX - hba + 1;
        }

        unmap_mapping_pages(mapping, hba, hlen, even_cows);
}
EXPORT_SYMBOL(unmap_mapping_range);

/*
 * Restore a potential device exclusive pte to a working pte entry
 */
static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
{
        struct folio *folio = page_folio(vmf->page);
        struct vm_area_struct *vma = vmf->vma;
        struct mmu_notifier_range range;
        vm_fault_t ret;

        /*
         * We need a reference to lock the folio because we don't hold
         * the PTL so a racing thread can remove the device-exclusive
         * entry and unmap it. If the folio is free the entry must
         * have been removed already. If it happens to have already
         * been re-allocated after being freed all we do is lock and
         * unlock it.
         */
        if (!folio_try_get(folio))
                return 0;

        ret = folio_lock_or_retry(folio, vmf);
        if (ret) {
                folio_put(folio);
                return ret;
        }
        mmu_notifier_range_init_owner(&range, MMU_NOTIFY_CLEAR, 0,
                                vma->vm_mm, vmf->address & PAGE_MASK,
                                (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
        mmu_notifier_invalidate_range_start(&range);

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
                                &vmf->ptl);
        if (likely(vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                restore_exclusive_pte(vma, folio, vmf->page, vmf->address,
                                      vmf->pte, vmf->orig_pte);

        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        folio_unlock(folio);
        folio_put(folio);

        mmu_notifier_invalidate_range_end(&range);
        return 0;
}

static inline bool should_try_to_free_swap(struct folio *folio,
                                           struct vm_area_struct *vma,
                                           unsigned int fault_flags)
{
        if (!folio_test_swapcache(folio))
                return false;
        if (mem_cgroup_swap_full(folio) || (vma->vm_flags & VM_LOCKED) ||
            folio_test_mlocked(folio))
                return true;
        /*
         * If we want to map a page that's in the swapcache writable, we
         * have to detect via the refcount if we're really the exclusive
         * user. Try freeing the swapcache to get rid of the swapcache
         * reference only in case it's likely that we'll be the exlusive user.
         */
        return (fault_flags & FAULT_FLAG_WRITE) && !folio_test_ksm(folio) &&
                folio_ref_count(folio) == (1 + folio_nr_pages(folio));
}

static vm_fault_t pte_marker_clear(struct vm_fault *vmf)
{
        vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                       vmf->address, &vmf->ptl);
        if (!vmf->pte)
                return 0;
        /*
         * Be careful so that we will only recover a special uffd-wp pte into a
         * none pte.  Otherwise it means the pte could have changed, so retry.
         *
         * This should also cover the case where e.g. the pte changed
         * quickly from a PTE_MARKER_UFFD_WP into PTE_MARKER_POISONED.
         * So is_pte_marker() check is not enough to safely drop the pte.
         */
        if (pte_same(vmf->orig_pte, ptep_get(vmf->pte)))
                pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return 0;
}

static vm_fault_t do_pte_missing(struct vm_fault *vmf)
{
        if (vma_is_anonymous(vmf->vma))
                return do_anonymous_page(vmf);
        else
                return do_fault(vmf);
}

/*
 * This is actually a page-missing access, but with uffd-wp special pte
 * installed.  It means this pte was wr-protected before being unmapped.
 */
static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
{
        /*
         * Just in case there're leftover special ptes even after the region
         * got unregistered - we can simply clear them.
         */
        if (unlikely(!userfaultfd_wp(vmf->vma)))
                return pte_marker_clear(vmf);

        return do_pte_missing(vmf);
}

static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
{
        swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
        unsigned long marker = pte_marker_get(entry);

        /*
         * PTE markers should never be empty.  If anything weird happened,
         * the best thing to do is to kill the process along with its mm.
         */
        if (WARN_ON_ONCE(!marker))
                return VM_FAULT_SIGBUS;

        /* Higher priority than uffd-wp when data corrupted */
        if (marker & PTE_MARKER_POISONED)
                return VM_FAULT_HWPOISON;

        /* Hitting a guard page is always a fatal condition. */
        if (marker & PTE_MARKER_GUARD)
                return VM_FAULT_SIGSEGV;

        if (pte_marker_entry_uffd_wp(entry))
                return pte_marker_handle_uffd_wp(vmf);

        /* This is an unknown pte marker */
        return VM_FAULT_SIGBUS;
}

static struct folio *__alloc_swap_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        swp_entry_t entry;

        folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, vmf->address);
        if (!folio)
                return NULL;

        entry = pte_to_swp_entry(vmf->orig_pte);
        if (mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
                                           GFP_KERNEL, entry)) {
                folio_put(folio);
                return NULL;
        }

        return folio;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int non_swapcache_batch(swp_entry_t entry, int max_nr)
{
        struct swap_info_struct *si = swp_swap_info(entry);
        pgoff_t offset = swp_offset(entry);
        int i;

        /*
         * While allocating a large folio and doing swap_read_folio, which is
         * the case the being faulted pte doesn't have swapcache. We need to
         * ensure all PTEs have no cache as well, otherwise, we might go to
         * swap devices while the content is in swapcache.
         */
        for (i = 0; i < max_nr; i++) {
                if ((si->swap_map[offset + i] & SWAP_HAS_CACHE))
                        return i;
        }

        return i;
}

/*
 * Check if the PTEs within a range are contiguous swap entries
 * and have consistent swapcache, zeromap.
 */
static bool can_swapin_thp(struct vm_fault *vmf, pte_t *ptep, int nr_pages)
{
        unsigned long addr;
        swp_entry_t entry;
        int idx;
        pte_t pte;

        addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);
        idx = (vmf->address - addr) / PAGE_SIZE;
        pte = ptep_get(ptep);

        if (!pte_same(pte, pte_move_swp_offset(vmf->orig_pte, -idx)))
                return false;
        entry = pte_to_swp_entry(pte);
        if (swap_pte_batch(ptep, nr_pages, pte) != nr_pages)
                return false;

        /*
         * swap_read_folio() can't handle the case a large folio is hybridly
         * from different backends. And they are likely corner cases. Similar
         * things might be added once zswap support large folios.
         */
        if (unlikely(swap_zeromap_batch(entry, nr_pages, NULL) != nr_pages))
                return false;
        if (unlikely(non_swapcache_batch(entry, nr_pages) != nr_pages))
                return false;

        return true;
}

static inline unsigned long thp_swap_suitable_orders(pgoff_t swp_offset,
                                                     unsigned long addr,
                                                     unsigned long orders)
{
        int order, nr;

        order = highest_order(orders);

        /*
         * To swap in a THP with nr pages, we require that its first swap_offset
         * is aligned with that number, as it was when the THP was swapped out.
         * This helps filter out most invalid entries.
         */
        while (orders) {
                nr = 1 << order;
                if ((addr >> PAGE_SHIFT) % nr == swp_offset % nr)
                        break;
                order = next_order(&orders, order);
        }

        return orders;
}

static struct folio *alloc_swap_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long orders;
        struct folio *folio;
        unsigned long addr;
        swp_entry_t entry;
        spinlock_t *ptl;
        pte_t *pte;
        gfp_t gfp;
        int order;

        /*
         * If uffd is active for the vma we need per-page fault fidelity to
         * maintain the uffd semantics.
         */
        if (unlikely(userfaultfd_armed(vma)))
                goto fallback;

        /*
         * A large swapped out folio could be partially or fully in zswap. We
         * lack handling for such cases, so fallback to swapping in order-0
         * folio.
         */
        if (!zswap_never_enabled())
                goto fallback;

        entry = pte_to_swp_entry(vmf->orig_pte);
        /*
         * Get a list of all the (large) orders below PMD_ORDER that are enabled
         * and suitable for swapping THP.
         */
        orders = thp_vma_allowable_orders(vma, vma->vm_flags,
                        TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
        orders = thp_vma_suitable_orders(vma, vmf->address, orders);
        orders = thp_swap_suitable_orders(swp_offset(entry),
                                          vmf->address, orders);

        if (!orders)
                goto fallback;

        pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                  vmf->address & PMD_MASK, &ptl);
        if (unlikely(!pte))
                goto fallback;

        /*
         * For do_swap_page, find the highest order where the aligned range is
         * completely swap entries with contiguous swap offsets.
         */
        order = highest_order(orders);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                if (can_swapin_thp(vmf, pte + pte_index(addr), 1 << order))
                        break;
                order = next_order(&orders, order);
        }

        pte_unmap_unlock(pte, ptl);

        /* Try allocating the highest of the remaining orders. */
        gfp = vma_thp_gfp_mask(vma);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                folio = vma_alloc_folio(gfp, order, vma, addr);
                if (folio) {
                        if (!mem_cgroup_swapin_charge_folio(folio, vma->vm_mm,
                                                            gfp, entry))
                                return folio;
                        count_mthp_stat(order, MTHP_STAT_SWPIN_FALLBACK_CHARGE);
                        folio_put(folio);
                }
                count_mthp_stat(order, MTHP_STAT_SWPIN_FALLBACK);
                order = next_order(&orders, order);
        }

fallback:
        return __alloc_swap_folio(vmf);
}
#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
static struct folio *alloc_swap_folio(struct vm_fault *vmf)
{
        return __alloc_swap_folio(vmf);
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

static DECLARE_WAIT_QUEUE_HEAD(swapcache_wq);

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
 * We return with pte unmapped and unlocked.
 *
 * We return with the mmap_lock locked or unlocked in the same cases
 * as does filemap_fault().
 */
vm_fault_t do_swap_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *swapcache, *folio = NULL;
        DECLARE_WAITQUEUE(wait, current);
        struct page *page;
        struct swap_info_struct *si = NULL;
        rmap_t rmap_flags = RMAP_NONE;
        bool need_clear_cache = false;
        bool exclusive = false;
        swp_entry_t entry;
        pte_t pte;
        vm_fault_t ret = 0;
        void *shadow = NULL;
        int nr_pages;
        unsigned long page_idx;
        unsigned long address;
        pte_t *ptep;

        if (!pte_unmap_same(vmf))
                goto out;

        entry = pte_to_swp_entry(vmf->orig_pte);
        if (unlikely(non_swap_entry(entry))) {
                if (is_migration_entry(entry)) {
                        migration_entry_wait(vma->vm_mm, vmf->pmd,
                                             vmf->address);
                } else if (is_device_exclusive_entry(entry)) {
                        vmf->page = pfn_swap_entry_to_page(entry);
                        ret = remove_device_exclusive_entry(vmf);
                } else if (is_device_private_entry(entry)) {
                        if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
                                /*
                                 * migrate_to_ram is not yet ready to operate
                                 * under VMA lock.
                                 */
                                vma_end_read(vma);
                                ret = VM_FAULT_RETRY;
                                goto out;
                        }

                        vmf->page = pfn_swap_entry_to_page(entry);
                        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                        vmf->address, &vmf->ptl);
                        if (unlikely(!vmf->pte ||
                                     !pte_same(ptep_get(vmf->pte),
                                                        vmf->orig_pte)))
                                goto unlock;

                        /*
                         * Get a page reference while we know the page can't be
                         * freed.
                         */
                        if (trylock_page(vmf->page)) {
                                struct dev_pagemap *pgmap;

                                get_page(vmf->page);
                                pte_unmap_unlock(vmf->pte, vmf->ptl);
                                pgmap = page_pgmap(vmf->page);
                                ret = pgmap->ops->migrate_to_ram(vmf);
                                unlock_page(vmf->page);
                                put_page(vmf->page);
                        } else {
                                pte_unmap_unlock(vmf->pte, vmf->ptl);
                        }
                } else if (is_hwpoison_entry(entry)) {
                        ret = VM_FAULT_HWPOISON;
                } else if (is_pte_marker_entry(entry)) {
                        ret = handle_pte_marker(vmf);
                } else {
                        print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
                        ret = VM_FAULT_SIGBUS;
                }
                goto out;
        }

        /* Prevent swapoff from happening to us. */
        si = get_swap_device(entry);
        if (unlikely(!si))
                goto out;

        folio = swap_cache_get_folio(entry, vma, vmf->address);
        if (folio)
                page = folio_file_page(folio, swp_offset(entry));
        swapcache = folio;

        if (!folio) {
                if (data_race(si->flags & SWP_SYNCHRONOUS_IO) &&
                    __swap_count(entry) == 1) {
                        /* skip swapcache */
                        folio = alloc_swap_folio(vmf);
                        if (folio) {
                                __folio_set_locked(folio);
                                __folio_set_swapbacked(folio);

                                nr_pages = folio_nr_pages(folio);
                                if (folio_test_large(folio))
                                        entry.val = ALIGN_DOWN(entry.val, nr_pages);
                                /*
                                 * Prevent parallel swapin from proceeding with
                                 * the cache flag. Otherwise, another thread
                                 * may finish swapin first, free the entry, and
                                 * swapout reusing the same entry. It's
                                 * undetectable as pte_same() returns true due
                                 * to entry reuse.
                                 */
                                if (swapcache_prepare(entry, nr_pages)) {
                                        /*
                                         * Relax a bit to prevent rapid
                                         * repeated page faults.
                                         */
                                        add_wait_queue(&swapcache_wq, &wait);
                                        schedule_timeout_uninterruptible(1);
                                        remove_wait_queue(&swapcache_wq, &wait);
                                        goto out_page;
                                }
                                need_clear_cache = true;

                                memcg1_swapin(entry, nr_pages);

                                shadow = get_shadow_from_swap_cache(entry);
                                if (shadow)
                                        workingset_refault(folio, shadow);

                                folio_add_lru(folio);

                                /* To provide entry to swap_read_folio() */
                                folio->swap = entry;
                                swap_read_folio(folio, NULL);
                                folio->private = NULL;
                        }
                } else {
                        folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
                                                vmf);
                        swapcache = folio;
                }

                if (!folio) {
                        /*
                         * Back out if somebody else faulted in this pte
                         * while we released the pte lock.
                         */
                        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                        vmf->address, &vmf->ptl);
                        if (likely(vmf->pte &&
                                   pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                                ret = VM_FAULT_OOM;
                        goto unlock;
                }

                /* Had to read the page from swap area: Major fault */
                ret = VM_FAULT_MAJOR;
                count_vm_event(PGMAJFAULT);
                count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
                page = folio_file_page(folio, swp_offset(entry));
        } else if (PageHWPoison(page)) {
                /*
                 * hwpoisoned dirty swapcache pages are kept for killing
                 * owner processes (which may be unknown at hwpoison time)
                 */
                ret = VM_FAULT_HWPOISON;
                goto out_release;
        }

        ret |= folio_lock_or_retry(folio, vmf);
        if (ret & VM_FAULT_RETRY)
                goto out_release;

        if (swapcache) {
                /*
                 * Make sure folio_free_swap() or swapoff did not release the
                 * swapcache from under us.  The page pin, and pte_same test
                 * below, are not enough to exclude that.  Even if it is still
                 * swapcache, we need to check that the page's swap has not
                 * changed.
                 */
                if (unlikely(!folio_test_swapcache(folio) ||
                             page_swap_entry(page).val != entry.val))
                        goto out_page;

                /*
                 * KSM sometimes has to copy on read faults, for example, if
                 * folio->index of non-ksm folios would be nonlinear inside the
                 * anon VMA -- the ksm flag is lost on actual swapout.
                 */
                folio = ksm_might_need_to_copy(folio, vma, vmf->address);
                if (unlikely(!folio)) {
                        ret = VM_FAULT_OOM;
                        folio = swapcache;
                        goto out_page;
                } else if (unlikely(folio == ERR_PTR(-EHWPOISON))) {
                        ret = VM_FAULT_HWPOISON;
                        folio = swapcache;
                        goto out_page;
                }
                if (folio != swapcache)
                        page = folio_page(folio, 0);

                /*
                 * If we want to map a page that's in the swapcache writable, we
                 * have to detect via the refcount if we're really the exclusive
                 * owner. Try removing the extra reference from the local LRU
                 * caches if required.
                 */
                if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
                    !folio_test_ksm(folio) && !folio_test_lru(folio))
                        lru_add_drain();
        }

        folio_throttle_swaprate(folio, GFP_KERNEL);

        /*
         * Back out if somebody else already faulted in this pte.
         */
        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
                        &vmf->ptl);
        if (unlikely(!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)))
                goto out_nomap;

        if (unlikely(!folio_test_uptodate(folio))) {
                ret = VM_FAULT_SIGBUS;
                goto out_nomap;
        }

        /* allocated large folios for SWP_SYNCHRONOUS_IO */
        if (folio_test_large(folio) && !folio_test_swapcache(folio)) {
                unsigned long nr = folio_nr_pages(folio);
                unsigned long folio_start = ALIGN_DOWN(vmf->address, nr * PAGE_SIZE);
                unsigned long idx = (vmf->address - folio_start) / PAGE_SIZE;
                pte_t *folio_ptep = vmf->pte - idx;
                pte_t folio_pte = ptep_get(folio_ptep);

                if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
                    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
                        goto out_nomap;

                page_idx = idx;
                address = folio_start;
                ptep = folio_ptep;
                goto check_folio;
        }

        nr_pages = 1;
        page_idx = 0;
        address = vmf->address;
        ptep = vmf->pte;
        if (folio_test_large(folio) && folio_test_swapcache(folio)) {
                int nr = folio_nr_pages(folio);
                unsigned long idx = folio_page_idx(folio, page);
                unsigned long folio_start = address - idx * PAGE_SIZE;
                unsigned long folio_end = folio_start + nr * PAGE_SIZE;
                pte_t *folio_ptep;
                pte_t folio_pte;

                if (unlikely(folio_start < max(address & PMD_MASK, vma->vm_start)))
                        goto check_folio;
                if (unlikely(folio_end > pmd_addr_end(address, vma->vm_end)))
                        goto check_folio;

                folio_ptep = vmf->pte - idx;
                folio_pte = ptep_get(folio_ptep);
                if (!pte_same(folio_pte, pte_move_swp_offset(vmf->orig_pte, -idx)) ||
                    swap_pte_batch(folio_ptep, nr, folio_pte) != nr)
                        goto check_folio;

                page_idx = idx;
                address = folio_start;
                ptep = folio_ptep;
                nr_pages = nr;
                entry = folio->swap;
                page = &folio->page;
        }

check_folio:
        /*
         * PG_anon_exclusive reuses PG_mappedtodisk for anon pages. A swap pte
         * must never point at an anonymous page in the swapcache that is
         * PG_anon_exclusive. Sanity check that this holds and especially, that
         * no filesystem set PG_mappedtodisk on a page in the swapcache. Sanity
         * check after taking the PT lock and making sure that nobody
         * concurrently faulted in this page and set PG_anon_exclusive.
         */
        BUG_ON(!folio_test_anon(folio) && folio_test_mappedtodisk(folio));
        BUG_ON(folio_test_anon(folio) && PageAnonExclusive(page));

        /*
         * Check under PT lock (to protect against concurrent fork() sharing
         * the swap entry concurrently) for certainly exclusive pages.
         */
        if (!folio_test_ksm(folio)) {
                exclusive = pte_swp_exclusive(vmf->orig_pte);
                if (folio != swapcache) {
                        /*
                         * We have a fresh page that is not exposed to the
                         * swapcache -> certainly exclusive.
                         */
                        exclusive = true;
                } else if (exclusive && folio_test_writeback(folio) &&
                          data_race(si->flags & SWP_STABLE_WRITES)) {
                        /*
                         * This is tricky: not all swap backends support
                         * concurrent page modifications while under writeback.
                         *
                         * So if we stumble over such a page in the swapcache
                         * we must not set the page exclusive, otherwise we can
                         * map it writable without further checks and modify it
                         * while still under writeback.
                         *
                         * For these problematic swap backends, simply drop the
                         * exclusive marker: this is perfectly fine as we start
                         * writeback only if we fully unmapped the page and
                         * there are no unexpected references on the page after
                         * unmapping succeeded. After fully unmapped, no
                         * further GUP references (FOLL_GET and FOLL_PIN) can
                         * appear, so dropping the exclusive marker and mapping
                         * it only R/O is fine.
                         */
                        exclusive = false;
                }
        }

        /*
         * Some architectures may have to restore extra metadata to the page
         * when reading from swap. This metadata may be indexed by swap entry
         * so this must be called before swap_free().
         */
        arch_swap_restore(folio_swap(entry, folio), folio);

        /*
         * Remove the swap entry and conditionally try to free up the swapcache.
         * We're already holding a reference on the page but haven't mapped it
         * yet.
         */
        swap_free_nr(entry, nr_pages);
        if (should_try_to_free_swap(folio, vma, vmf->flags))
                folio_free_swap(folio);

        add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
        add_mm_counter(vma->vm_mm, MM_SWAPENTS, -nr_pages);
        pte = mk_pte(page, vma->vm_page_prot);
        if (pte_swp_soft_dirty(vmf->orig_pte))
                pte = pte_mksoft_dirty(pte);
        if (pte_swp_uffd_wp(vmf->orig_pte))
                pte = pte_mkuffd_wp(pte);

        /*
         * Same logic as in do_wp_page(); however, optimize for pages that are
         * certainly not shared either because we just allocated them without
         * exposing them to the swapcache or because the swap entry indicates
         * exclusivity.
         */
        if (!folio_test_ksm(folio) &&
            (exclusive || folio_ref_count(folio) == 1)) {
                if ((vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) &&
                    !pte_needs_soft_dirty_wp(vma, pte)) {
                        pte = pte_mkwrite(pte, vma);
                        if (vmf->flags & FAULT_FLAG_WRITE) {
                                pte = pte_mkdirty(pte);
                                vmf->flags &= ~FAULT_FLAG_WRITE;
                        }
                }
                rmap_flags |= RMAP_EXCLUSIVE;
        }
        folio_ref_add(folio, nr_pages - 1);
        flush_icache_pages(vma, page, nr_pages);
        vmf->orig_pte = pte_advance_pfn(pte, page_idx);

        /* ksm created a completely new copy */
        if (unlikely(folio != swapcache && swapcache)) {
                folio_add_new_anon_rmap(folio, vma, address, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else if (!folio_test_anon(folio)) {
                /*
                 * We currently only expect small !anon folios which are either
                 * fully exclusive or fully shared, or new allocated large
                 * folios which are fully exclusive. If we ever get large
                 * folios within swapcache here, we have to be careful.
                 */
                VM_WARN_ON_ONCE(folio_test_large(folio) && folio_test_swapcache(folio));
                VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio);
                folio_add_new_anon_rmap(folio, vma, address, rmap_flags);
        } else {
                folio_add_anon_rmap_ptes(folio, page, nr_pages, vma, address,
                                        rmap_flags);
        }

        VM_BUG_ON(!folio_test_anon(folio) ||
                        (pte_write(pte) && !PageAnonExclusive(page)));
        set_ptes(vma->vm_mm, address, ptep, pte, nr_pages);
        arch_do_swap_page_nr(vma->vm_mm, vma, address,
                        pte, pte, nr_pages);

        folio_unlock(folio);
        if (folio != swapcache && swapcache) {
                /*
                 * Hold the lock to avoid the swap entry to be reused
                 * until we take the PT lock for the pte_same() check
                 * (to avoid false positives from pte_same). For
                 * further safety release the lock after the swap_free
                 * so that the swap count won't change under a
                 * parallel locked swapcache.
                 */
                folio_unlock(swapcache);
                folio_put(swapcache);
        }

        if (vmf->flags & FAULT_FLAG_WRITE) {
                ret |= do_wp_page(vmf);
                if (ret & VM_FAULT_ERROR)
                        ret &= VM_FAULT_ERROR;
                goto out;
        }

        /* No need to invalidate - it was non-present before */
        update_mmu_cache_range(vmf, vma, address, ptep, nr_pages);
unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
out:
        /* Clear the swap cache pin for direct swapin after PTL unlock */
        if (need_clear_cache) {
                swapcache_clear(si, entry, nr_pages);
                if (waitqueue_active(&swapcache_wq))
                        wake_up(&swapcache_wq);
        }
        if (si)
                put_swap_device(si);
        return ret;
out_nomap:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
out_page:
        folio_unlock(folio);
out_release:
        folio_put(folio);
        if (folio != swapcache && swapcache) {
                folio_unlock(swapcache);
                folio_put(swapcache);
        }
        if (need_clear_cache) {
                swapcache_clear(si, entry, nr_pages);
                if (waitqueue_active(&swapcache_wq))
                        wake_up(&swapcache_wq);
        }
        if (si)
                put_swap_device(si);
        return ret;
}

static bool pte_range_none(pte_t *pte, int nr_pages)
{
        int i;

        for (i = 0; i < nr_pages; i++) {
                if (!pte_none(ptep_get_lockless(pte + i)))
                        return false;
        }

        return true;
}

static struct folio *alloc_anon_folio(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
        unsigned long orders;
        struct folio *folio;
        unsigned long addr;
        pte_t *pte;
        gfp_t gfp;
        int order;

        /*
         * If uffd is active for the vma we need per-page fault fidelity to
         * maintain the uffd semantics.
         */
        if (unlikely(userfaultfd_armed(vma)))
                goto fallback;

        /*
         * Get a list of all the (large) orders below PMD_ORDER that are enabled
         * for this vma. Then filter out the orders that can't be allocated over
         * the faulting address and still be fully contained in the vma.
         */
        orders = thp_vma_allowable_orders(vma, vma->vm_flags,
                        TVA_IN_PF | TVA_ENFORCE_SYSFS, BIT(PMD_ORDER) - 1);
        orders = thp_vma_suitable_orders(vma, vmf->address, orders);

        if (!orders)
                goto fallback;

        pte = pte_offset_map(vmf->pmd, vmf->address & PMD_MASK);
        if (!pte)
                return ERR_PTR(-EAGAIN);

        /*
         * Find the highest order where the aligned range is completely
         * pte_none(). Note that all remaining orders will be completely
         * pte_none().
         */
        order = highest_order(orders);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                if (pte_range_none(pte + pte_index(addr), 1 << order))
                        break;
                order = next_order(&orders, order);
        }

        pte_unmap(pte);

        if (!orders)
                goto fallback;

        /* Try allocating the highest of the remaining orders. */
        gfp = vma_thp_gfp_mask(vma);
        while (orders) {
                addr = ALIGN_DOWN(vmf->address, PAGE_SIZE << order);
                folio = vma_alloc_folio(gfp, order, vma, addr);
                if (folio) {
                        if (mem_cgroup_charge(folio, vma->vm_mm, gfp)) {
                                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
                                folio_put(folio);
                                goto next;
                        }
                        folio_throttle_swaprate(folio, gfp);
                        /*
                         * When a folio is not zeroed during allocation
                         * (__GFP_ZERO not used) or user folios require special
                         * handling, folio_zero_user() is used to make sure
                         * that the page corresponding to the faulting address
                         * will be hot in the cache after zeroing.
                         */
                        if (user_alloc_needs_zeroing())
                                folio_zero_user(folio, vmf->address);
                        return folio;
                }
next:
                count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK);
                order = next_order(&orders, order);
        }

fallback:
#endif
        return folio_prealloc(vma->vm_mm, vma, vmf->address, true);
}

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults), and pte mapped but not yet locked.
 * We return with mmap_lock still held, but pte unmapped and unlocked.
 */
static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        unsigned long addr = vmf->address;
        struct folio *folio;
        vm_fault_t ret = 0;
        int nr_pages = 1;
        pte_t entry;

        /* File mapping without ->vm_ops ? */
        if (vma->vm_flags & VM_SHARED)
                return VM_FAULT_SIGBUS;

        /*
         * Use pte_alloc() instead of pte_alloc_map(), so that OOM can
         * be distinguished from a transient failure of pte_offset_map().
         */
        if (pte_alloc(vma->vm_mm, vmf->pmd))
                return VM_FAULT_OOM;

        /* Use the zero-page for reads */
        if (!(vmf->flags & FAULT_FLAG_WRITE) &&
                        !mm_forbids_zeropage(vma->vm_mm)) {
                entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
                                                vma->vm_page_prot));
                vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                vmf->address, &vmf->ptl);
                if (!vmf->pte)
                        goto unlock;
                if (vmf_pte_changed(vmf)) {
                        update_mmu_tlb(vma, vmf->address, vmf->pte);
                        goto unlock;
                }
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        goto unlock;
                /* Deliver the page fault to userland, check inside PT lock */
                if (userfaultfd_missing(vma)) {
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                        return handle_userfault(vmf, VM_UFFD_MISSING);
                }
                goto setpte;
        }

        /* Allocate our own private page. */
        ret = vmf_anon_prepare(vmf);
        if (ret)
                return ret;
        /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */
        folio = alloc_anon_folio(vmf);
        if (IS_ERR(folio))
                return 0;
        if (!folio)
                goto oom;

        nr_pages = folio_nr_pages(folio);
        addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE);

        /*
         * The memory barrier inside __folio_mark_uptodate makes sure that
         * preceding stores to the page contents become visible before
         * the set_pte_at() write.
         */
        __folio_mark_uptodate(folio);

        entry = folio_mk_pte(folio, vma->vm_page_prot);
        entry = pte_sw_mkyoung(entry);
        if (vma->vm_flags & VM_WRITE)
                entry = pte_mkwrite(pte_mkdirty(entry), vma);

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
        if (!vmf->pte)
                goto release;
        if (nr_pages == 1 && vmf_pte_changed(vmf)) {
                update_mmu_tlb(vma, addr, vmf->pte);
                goto release;
        } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
                update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
                goto release;
        }

        ret = check_stable_address_space(vma->vm_mm);
        if (ret)
                goto release;

        /* Deliver the page fault to userland, check inside PT lock */
        if (userfaultfd_missing(vma)) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                folio_put(folio);
                return handle_userfault(vmf, VM_UFFD_MISSING);
        }

        folio_ref_add(folio, nr_pages - 1);
        add_mm_counter(vma->vm_mm, MM_ANONPAGES, nr_pages);
        count_mthp_stat(folio_order(folio), MTHP_STAT_ANON_FAULT_ALLOC);
        folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
        folio_add_lru_vma(folio, vma);
setpte:
        if (vmf_orig_pte_uffd_wp(vmf))
                entry = pte_mkuffd_wp(entry);
        set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr_pages);

        /* No need to invalidate - it was non-present before */
        update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr_pages);
unlock:
        if (vmf->pte)
                pte_unmap_unlock(vmf->pte, vmf->ptl);
        return ret;
release:
        folio_put(folio);
        goto unlock;
oom:
        return VM_FAULT_OOM;
}

/*
 * The mmap_lock must have been held on entry, and may have been
 * released depending on flags and vma->vm_ops->fault() return value.
 * See filemap_fault() and __lock_page_retry().
 */
static vm_fault_t __do_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        vm_fault_t ret;

        /*
         * Preallocate pte before we take page_lock because this might lead to
         * deadlocks for memcg reclaim which waits for pages under writeback:
         *                                lock_page(A)
         *                                SetPageWriteback(A)
         *                                unlock_page(A)
         * lock_page(B)
         *                                lock_page(B)
         * pte_alloc_one
         *   shrink_folio_list
         *     wait_on_page_writeback(A)
         *                                SetPageWriteback(B)
         *                                unlock_page(B)
         *                                # flush A, B to clear the writeback
         */
        if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) {
                vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        ret = vma->vm_ops->fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY |
                            VM_FAULT_DONE_COW)))
                return ret;

        folio = page_folio(vmf->page);
        if (unlikely(PageHWPoison(vmf->page))) {
                vm_fault_t poisonret = VM_FAULT_HWPOISON;
                if (ret & VM_FAULT_LOCKED) {
                        if (page_mapped(vmf->page))
                                unmap_mapping_folio(folio);
                        /* Retry if a clean folio was removed from the cache. */
                        if (mapping_evict_folio(folio->mapping, folio))
                                poisonret = VM_FAULT_NOPAGE;
                        folio_unlock(folio);
                }
                folio_put(folio);
                vmf->page = NULL;
                return poisonret;
        }

        if (unlikely(!(ret & VM_FAULT_LOCKED)))
                folio_lock(folio);
        else
                VM_BUG_ON_PAGE(!folio_test_locked(folio), vmf->page);

        return ret;
}

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void deposit_prealloc_pte(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;

        pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
        /*
         * We are going to consume the prealloc table,
         * count that as nr_ptes.
         */
        mm_inc_nr_ptes(vma->vm_mm);
        vmf->prealloc_pte = NULL;
}

vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page)
{
        struct vm_area_struct *vma = vmf->vma;
        bool write = vmf->flags & FAULT_FLAG_WRITE;
        unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
        pmd_t entry;
        vm_fault_t ret = VM_FAULT_FALLBACK;

        /*
         * It is too late to allocate a small folio, we already have a large
         * folio in the pagecache: especially s390 KVM cannot tolerate any
         * PMD mappings, but PTE-mapped THP are fine. So let's simply refuse any
         * PMD mappings if THPs are disabled.
         */
        if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags))
                return ret;

        if (!thp_vma_suitable_order(vma, haddr, PMD_ORDER))
                return ret;

        if (folio_order(folio) != HPAGE_PMD_ORDER)
                return ret;
        page = &folio->page;

        /*
         * Just backoff if any subpage of a THP is corrupted otherwise
         * the corrupted page may mapped by PMD silently to escape the
         * check.  This kind of THP just can be PTE mapped.  Access to
         * the corrupted subpage should trigger SIGBUS as expected.
         */
        if (unlikely(folio_test_has_hwpoisoned(folio)))
                return ret;

        /*
         * Archs like ppc64 need additional space to store information
         * related to pte entry. Use the preallocated table for that.
         */
        if (arch_needs_pgtable_deposit() && !vmf->prealloc_pte) {
                vmf->prealloc_pte = pte_alloc_one(vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
        if (unlikely(!pmd_none(*vmf->pmd)))
                goto out;

        flush_icache_pages(vma, page, HPAGE_PMD_NR);

        entry = folio_mk_pmd(folio, vma->vm_page_prot);
        if (write)
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);

        add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR);
        folio_add_file_rmap_pmd(folio, page, vma);

        /*
         * deposit and withdraw with pmd lock held
         */
        if (arch_needs_pgtable_deposit())
                deposit_prealloc_pte(vmf);

        set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);

        update_mmu_cache_pmd(vma, haddr, vmf->pmd);

        /* fault is handled */
        ret = 0;
        count_vm_event(THP_FILE_MAPPED);
out:
        spin_unlock(vmf->ptl);
        return ret;
}
#else
vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *page)
{
        return VM_FAULT_FALLBACK;
}
#endif

/**
 * set_pte_range - Set a range of PTEs to point to pages in a folio.
 * @vmf: Fault decription.
 * @folio: The folio that contains @page.
 * @page: The first page to create a PTE for.
 * @nr: The number of PTEs to create.
 * @addr: The first address to create a PTE for.
 */
void set_pte_range(struct vm_fault *vmf, struct folio *folio,
                struct page *page, unsigned int nr, unsigned long addr)
{
        struct vm_area_struct *vma = vmf->vma;
        bool write = vmf->flags & FAULT_FLAG_WRITE;
        bool prefault = !in_range(vmf->address, addr, nr * PAGE_SIZE);
        pte_t entry;

        flush_icache_pages(vma, page, nr);
        entry = mk_pte(page, vma->vm_page_prot);

        if (prefault && arch_wants_old_prefaulted_pte())
                entry = pte_mkold(entry);
        else
                entry = pte_sw_mkyoung(entry);

        if (write)
                entry = maybe_mkwrite(pte_mkdirty(entry), vma);
        else if (pte_write(entry) && folio_test_dirty(folio))
                entry = pte_mkdirty(entry);
        if (unlikely(vmf_orig_pte_uffd_wp(vmf)))
                entry = pte_mkuffd_wp(entry);
        /* copy-on-write page */
        if (write && !(vma->vm_flags & VM_SHARED)) {
                VM_BUG_ON_FOLIO(nr != 1, folio);
                folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE);
                folio_add_lru_vma(folio, vma);
        } else {
                folio_add_file_rmap_ptes(folio, page, nr, vma);
        }
        set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr);

        /* no need to invalidate: a not-present page won't be cached */
        update_mmu_cache_range(vmf, vma, addr, vmf->pte, nr);
}

static bool vmf_pte_changed(struct vm_fault *vmf)
{
        if (vmf->flags & FAULT_FLAG_ORIG_PTE_VALID)
                return !pte_same(ptep_get(vmf->pte), vmf->orig_pte);

        return !pte_none(ptep_get(vmf->pte));
}

/**
 * finish_fault - finish page fault once we have prepared the page to fault
 *
 * @vmf: structure describing the fault
 *
 * This function handles all that is needed to finish a page fault once the
 * page to fault in is prepared. It handles locking of PTEs, inserts PTE for
 * given page, adds reverse page mapping, handles memcg charges and LRU
 * addition.
 *
 * The function expects the page to be locked and on success it consumes a
 * reference of a page being mapped (for the PTE which maps it).
 *
 * Return: %0 on success, %VM_FAULT_ code in case of error.
 */
vm_fault_t finish_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct page *page;
        struct folio *folio;
        vm_fault_t ret;
        bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) &&
                      !(vma->vm_flags & VM_SHARED);
        int type, nr_pages;
        unsigned long addr;
        bool needs_fallback = false;

fallback:
        addr = vmf->address;

        /* Did we COW the page? */
        if (is_cow)
                page = vmf->cow_page;
        else
                page = vmf->page;

        folio = page_folio(page);
        /*
         * check even for read faults because we might have lost our CoWed
         * page
         */
        if (!(vma->vm_flags & VM_SHARED)) {
                ret = check_stable_address_space(vma->vm_mm);
                if (ret)
                        return ret;
        }

        if (pmd_none(*vmf->pmd)) {
                if (folio_test_pmd_mappable(folio)) {
                        ret = do_set_pmd(vmf, folio, page);
                        if (ret != VM_FAULT_FALLBACK)
                                return ret;
                }

                if (vmf->prealloc_pte)
                        pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte);
                else if (unlikely(pte_alloc(vma->vm_mm, vmf->pmd)))
                        return VM_FAULT_OOM;
        }

        nr_pages = folio_nr_pages(folio);

        /*
         * Using per-page fault to maintain the uffd semantics, and same
         * approach also applies to non-anonymous-shmem faults to avoid
         * inflating the RSS of the process.
         */
        if (!vma_is_anon_shmem(vma) || unlikely(userfaultfd_armed(vma)) ||
            unlikely(needs_fallback)) {
                nr_pages = 1;
        } else if (nr_pages > 1) {
                pgoff_t idx = folio_page_idx(folio, page);
                /* The page offset of vmf->address within the VMA. */
                pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
                /* The index of the entry in the pagetable for fault page. */
                pgoff_t pte_off = pte_index(vmf->address);

                /*
                 * Fallback to per-page fault in case the folio size in page
                 * cache beyond the VMA limits and PMD pagetable limits.
                 */
                if (unlikely(vma_off < idx ||
                            vma_off + (nr_pages - idx) > vma_pages(vma) ||
                            pte_off < idx ||
                            pte_off + (nr_pages - idx)  > PTRS_PER_PTE)) {
                        nr_pages = 1;
                } else {
                        /* Now we can set mappings for the whole large folio. */
                        addr = vmf->address - idx * PAGE_SIZE;
                        page = &folio->page;
                }
        }

        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                       addr, &vmf->ptl);
        if (!vmf->pte)
                return VM_FAULT_NOPAGE;

        /* Re-check under ptl */
        if (nr_pages == 1 && unlikely(vmf_pte_changed(vmf))) {
                update_mmu_tlb(vma, addr, vmf->pte);
                ret = VM_FAULT_NOPAGE;
                goto unlock;
        } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
                needs_fallback = true;
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                goto fallback;
        }

        folio_ref_add(folio, nr_pages - 1);
        set_pte_range(vmf, folio, page, nr_pages, addr);
        type = is_cow ? MM_ANONPAGES : mm_counter_file(folio);
        add_mm_counter(vma->vm_mm, type, nr_pages);
        ret = 0;

unlock:
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return ret;
}

static unsigned long fault_around_pages __read_mostly =
        65536 >> PAGE_SHIFT;

#ifdef CONFIG_DEBUG_FS
static int fault_around_bytes_get(void *data, u64 *val)
{
        *val = fault_around_pages << PAGE_SHIFT;
        return 0;
}

/*
 * fault_around_bytes must be rounded down to the nearest page order as it's
 * what do_fault_around() expects to see.
 */
static int fault_around_bytes_set(void *data, u64 val)
{
        if (val / PAGE_SIZE > PTRS_PER_PTE)
                return -EINVAL;

        /*
         * The minimum value is 1 page, however this results in no fault-around
         * at all. See should_fault_around().
         */
        val = max(val, PAGE_SIZE);
        fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT;

        return 0;
}
DEFINE_DEBUGFS_ATTRIBUTE(fault_around_bytes_fops,
                fault_around_bytes_get, fault_around_bytes_set, "%llu\n");

static int __init fault_around_debugfs(void)
{
        debugfs_create_file_unsafe("fault_around_bytes", 0644, NULL, NULL,
                                   &fault_around_bytes_fops);
        return 0;
}
late_initcall(fault_around_debugfs);
#endif

/*
 * do_fault_around() tries to map few pages around the fault address. The hope
 * is that the pages will be needed soon and this will lower the number of
 * faults to handle.
 *
 * It uses vm_ops->map_pages() to map the pages, which skips the page if it's
 * not ready to be mapped: not up-to-date, locked, etc.
 *
 * This function doesn't cross VMA or page table boundaries, in order to call
 * map_pages() and acquire a PTE lock only once.
 *
 * fault_around_pages defines how many pages we'll try to map.
 * do_fault_around() expects it to be set to a power of two less than or equal
 * to PTRS_PER_PTE.
 *
 * The virtual address of the area that we map is naturally aligned to
 * fault_around_pages * PAGE_SIZE rounded down to the machine page size
 * (and therefore to page order).  This way it's easier to guarantee
 * that we don't cross page table boundaries.
 */
static vm_fault_t do_fault_around(struct vm_fault *vmf)
{
        pgoff_t nr_pages = READ_ONCE(fault_around_pages);
        pgoff_t pte_off = pte_index(vmf->address);
        /* The page offset of vmf->address within the VMA. */
        pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff;
        pgoff_t from_pte, to_pte;
        vm_fault_t ret;

        /* The PTE offset of the start address, clamped to the VMA. */
        from_pte = max(ALIGN_DOWN(pte_off, nr_pages),
                       pte_off - min(pte_off, vma_off));

        /* The PTE offset of the end address, clamped to the VMA and PTE. */
        to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE,
                      pte_off + vma_pages(vmf->vma) - vma_off) - 1;

        if (pmd_none(*vmf->pmd)) {
                vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm);
                if (!vmf->prealloc_pte)
                        return VM_FAULT_OOM;
        }

        rcu_read_lock();
        ret = vmf->vma->vm_ops->map_pages(vmf,
                        vmf->pgoff + from_pte - pte_off,
                        vmf->pgoff + to_pte - pte_off);
        rcu_read_unlock();

        return ret;
}

/* Return true if we should do read fault-around, false otherwise */
static inline bool should_fault_around(struct vm_fault *vmf)
{
        /* No ->map_pages?  No way to fault around... */
        if (!vmf->vma->vm_ops->map_pages)
                return false;

        if (uffd_disable_fault_around(vmf->vma))
                return false;

        /* A single page implies no faulting 'around' at all. */
        return fault_around_pages > 1;
}

static vm_fault_t do_read_fault(struct vm_fault *vmf)
{
        vm_fault_t ret = 0;
        struct folio *folio;

        /*
         * Let's call ->map_pages() first and use ->fault() as fallback
         * if page by the offset is not ready to be mapped (cold cache or
         * something).
         */
        if (should_fault_around(vmf)) {
                ret = do_fault_around(vmf);
                if (ret)
                        return ret;
        }

        ret = vmf_can_call_fault(vmf);
        if (ret)
                return ret;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                return ret;

        ret |= finish_fault(vmf);
        folio = page_folio(vmf->page);
        folio_unlock(folio);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                folio_put(folio);
        return ret;
}

static vm_fault_t do_cow_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio;
        vm_fault_t ret;

        ret = vmf_can_call_fault(vmf);
        if (!ret)
                ret = vmf_anon_prepare(vmf);
        if (ret)
                return ret;

        folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false);
        if (!folio)
                return VM_FAULT_OOM;

        vmf->cow_page = &folio->page;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                goto uncharge_out;
        if (ret & VM_FAULT_DONE_COW)
                return ret;

        if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) {
                ret = VM_FAULT_HWPOISON;
                goto unlock;
        }
        __folio_mark_uptodate(folio);

        ret |= finish_fault(vmf);
unlock:
        unlock_page(vmf->page);
        put_page(vmf->page);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                goto uncharge_out;
        return ret;
uncharge_out:
        folio_put(folio);
        return ret;
}

static vm_fault_t do_shared_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret, tmp;
        struct folio *folio;

        ret = vmf_can_call_fault(vmf);
        if (ret)
                return ret;

        ret = __do_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
                return ret;

        folio = page_folio(vmf->page);

        /*
         * Check if the backing address space wants to know that the page is
         * about to become writable
         */
        if (vma->vm_ops->page_mkwrite) {
                folio_unlock(folio);
                tmp = do_page_mkwrite(vmf, folio);
                if (unlikely(!tmp ||
                                (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) {
                        folio_put(folio);
                        return tmp;
                }
        }

        ret |= finish_fault(vmf);
        if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE |
                                        VM_FAULT_RETRY))) {
                folio_unlock(folio);
                folio_put(folio);
                return ret;
        }

        ret |= fault_dirty_shared_page(vmf);
        return ret;
}

/*
 * We enter with non-exclusive mmap_lock (to exclude vma changes,
 * but allow concurrent faults).
 * The mmap_lock may have been released depending on flags and our
 * return value.  See filemap_fault() and __folio_lock_or_retry().
 * If mmap_lock is released, vma may become invalid (for example
 * by other thread calling munmap()).
 */
static vm_fault_t do_fault(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct mm_struct *vm_mm = vma->vm_mm;
        vm_fault_t ret;

        /*
         * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
         */
        if (!vma->vm_ops->fault) {
                vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
                                               vmf->address, &vmf->ptl);
                if (unlikely(!vmf->pte))
                        ret = VM_FAULT_SIGBUS;
                else {
                        /*
                         * Make sure this is not a temporary clearing of pte
                         * by holding ptl and checking again. A R/M/W update
                         * of pte involves: take ptl, clearing the pte so that
                         * we don't have concurrent modification by hardware
                         * followed by an update.
                         */
                        if (unlikely(pte_none(ptep_get(vmf->pte))))
                                ret = VM_FAULT_SIGBUS;
                        else
                                ret = VM_FAULT_NOPAGE;

                        pte_unmap_unlock(vmf->pte, vmf->ptl);
                }
        } else if (!(vmf->flags & FAULT_FLAG_WRITE))
                ret = do_read_fault(vmf);
        else if (!(vma->vm_flags & VM_SHARED))
                ret = do_cow_fault(vmf);
        else
                ret = do_shared_fault(vmf);

        /* preallocated pagetable is unused: free it */
        if (vmf->prealloc_pte) {
                pte_free(vm_mm, vmf->prealloc_pte);
                vmf->prealloc_pte = NULL;
        }
        return ret;
}

int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
                      unsigned long addr, int *flags,
                      bool writable, int *last_cpupid)
{
        struct vm_area_struct *vma = vmf->vma;

        /*
         * Avoid grouping on RO pages in general. RO pages shouldn't hurt as
         * much anyway since they can be in shared cache state. This misses
         * the case where a mapping is writable but the process never writes
         * to it but pte_write gets cleared during protection updates and
         * pte_dirty has unpredictable behaviour between PTE scan updates,
         * background writeback, dirty balancing and application behaviour.
         */
        if (!writable)
                *flags |= TNF_NO_GROUP;

        /*
         * Flag if the folio is shared between multiple address spaces. This
         * is later used when determining whether to group tasks together
         */
        if (folio_maybe_mapped_shared(folio) && (vma->vm_flags & VM_SHARED))
                *flags |= TNF_SHARED;
        /*
         * For memory tiering mode, cpupid of slow memory page is used
         * to record page access time.  So use default value.
         */
        if (folio_use_access_time(folio))
                *last_cpupid = (-1 & LAST_CPUPID_MASK);
        else
                *last_cpupid = folio_last_cpupid(folio);

        /* Record the current PID acceesing VMA */
        vma_set_access_pid_bit(vma);

        count_vm_numa_event(NUMA_HINT_FAULTS);
#ifdef CONFIG_NUMA_BALANCING
        count_memcg_folio_events(folio, NUMA_HINT_FAULTS, 1);
#endif
        if (folio_nid(folio) == numa_node_id()) {
                count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
                *flags |= TNF_FAULT_LOCAL;
        }

        return mpol_misplaced(folio, vmf, addr);
}

static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
                                        unsigned long fault_addr, pte_t *fault_pte,
                                        bool writable)
{
        pte_t pte, old_pte;

        old_pte = ptep_modify_prot_start(vma, fault_addr, fault_pte);
        pte = pte_modify(old_pte, vma->vm_page_prot);
        pte = pte_mkyoung(pte);
        if (writable)
                pte = pte_mkwrite(pte, vma);
        ptep_modify_prot_commit(vma, fault_addr, fault_pte, old_pte, pte);
        update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1);
}

static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma,
                                       struct folio *folio, pte_t fault_pte,
                                       bool ignore_writable, bool pte_write_upgrade)
{
        int nr = pte_pfn(fault_pte) - folio_pfn(folio);
        unsigned long start, end, addr = vmf->address;
        unsigned long addr_start = addr - (nr << PAGE_SHIFT);
        unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE);
        pte_t *start_ptep;

        /* Stay within the VMA and within the page table. */
        start = max3(addr_start, pt_start, vma->vm_start);
        end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE,
                   vma->vm_end);
        start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT);

        /* Restore all PTEs' mapping of the large folio */
        for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) {
                pte_t ptent = ptep_get(start_ptep);
                bool writable = false;

                if (!pte_present(ptent) || !pte_protnone(ptent))
                        continue;

                if (pfn_folio(pte_pfn(ptent)) != folio)
                        continue;

                if (!ignore_writable) {
                        ptent = pte_modify(ptent, vma->vm_page_prot);
                        writable = pte_write(ptent);
                        if (!writable && pte_write_upgrade &&
                            can_change_pte_writable(vma, addr, ptent))
                                writable = true;
                }

                numa_rebuild_single_mapping(vmf, vma, addr, start_ptep, writable);
        }
}

static vm_fault_t do_numa_page(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        struct folio *folio = NULL;
        int nid = NUMA_NO_NODE;
        bool writable = false, ignore_writable = false;
        bool pte_write_upgrade = vma_wants_manual_pte_write_upgrade(vma);
        int last_cpupid;
        int target_nid;
        pte_t pte, old_pte;
        int flags = 0, nr_pages;

        /*
         * The pte cannot be used safely until we verify, while holding the page
         * table lock, that its contents have not changed during fault handling.
         */
        spin_lock(vmf->ptl);
        /* Read the live PTE from the page tables: */
        old_pte = ptep_get(vmf->pte);

        if (unlikely(!pte_same(old_pte, vmf->orig_pte))) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return 0;
        }

        pte = pte_modify(old_pte, vma->vm_page_prot);

        /*
         * Detect now whether the PTE could be writable; this information
         * is only valid while holding the PT lock.
         */
        writable = pte_write(pte);
        if (!writable && pte_write_upgrade &&
            can_change_pte_writable(vma, vmf->address, pte))
                writable = true;

        folio = vm_normal_folio(vma, vmf->address, pte);
        if (!folio || folio_is_zone_device(folio))
                goto out_map;

        nid = folio_nid(folio);
        nr_pages = folio_nr_pages(folio);

        target_nid = numa_migrate_check(folio, vmf, vmf->address, &flags,
                                        writable, &last_cpupid);
        if (target_nid == NUMA_NO_NODE)
                goto out_map;
        if (migrate_misplaced_folio_prepare(folio, vma, target_nid)) {
                flags |= TNF_MIGRATE_FAIL;
                goto out_map;
        }
        /* The folio is isolated and isolation code holds a folio reference. */
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        writable = false;
        ignore_writable = true;

        /* Migrate to the requested node */
        if (!migrate_misplaced_folio(folio, target_nid)) {
                nid = target_nid;
                flags |= TNF_MIGRATED;
                task_numa_fault(last_cpupid, nid, nr_pages, flags);
                return 0;
        }

        flags |= TNF_MIGRATE_FAIL;
        vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
                                       vmf->address, &vmf->ptl);
        if (unlikely(!vmf->pte))
                return 0;
        if (unlikely(!pte_same(ptep_get(vmf->pte), vmf->orig_pte))) {
                pte_unmap_unlock(vmf->pte, vmf->ptl);
                return 0;
        }
out_map:
        /*
         * Make it present again, depending on how arch implements
         * non-accessible ptes, some can allow access by kernel mode.
         */
        if (folio && folio_test_large(folio))
                numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable,
                                           pte_write_upgrade);
        else
                numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte,
                                            writable);
        pte_unmap_unlock(vmf->pte, vmf->ptl);

        if (nid != NUMA_NO_NODE)
                task_numa_fault(last_cpupid, nid, nr_pages, flags);
        return 0;
}

static inline vm_fault_t create_huge_pmd(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        if (vma_is_anonymous(vma))
                return do_huge_pmd_anonymous_page(vmf);
        if (vma->vm_ops->huge_fault)
                return vma->vm_ops->huge_fault(vmf, PMD_ORDER);
        return VM_FAULT_FALLBACK;
}

/* `inline' is required to avoid gcc 4.1.2 build error */
static inline vm_fault_t wp_huge_pmd(struct vm_fault *vmf)
{
        struct vm_area_struct *vma = vmf->vma;
        const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE;
        vm_fault_t ret;

        if (vma_is_anonymous(vma)) {
                if (likely(!unshare) &&
                    userfaultfd_huge_pmd_wp(vma, vmf->orig_pmd)) {
                        if (userfaultfd_wp_async(vmf->vma))
                                goto split;
                        return handle_userfault(vmf, VM_UFFD_WP);
                }
                return do_huge_pmd_wp_page(vmf);
        }

        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                if (vma->vm_ops->huge_fault) {
                        ret = vma->vm_ops->huge_fault(vmf, PMD_ORDER);
                        if (!(ret & VM_FAULT_FALLBACK))
                                return ret;
                }
        }

split:
        /* COW or write-notify handled on pte level: split pmd. */
        __split_huge_pmd(vma, vmf->pmd, vmf->address, false);

        return VM_FAULT_FALLBACK;
}

static vm_fault_t create_huge_pud(struct vm_fault *vmf)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&                        \
        defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
        struct vm_area_struct *vma = vmf->vma;
        /* No support for anonymous transparent PUD pages yet */
        if (vma_is_anonymous(vma))
                return VM_FAULT_FALLBACK;
        if (vma->vm_ops->huge_fault)
                return vma->vm_ops->huge_fault(vmf, PUD_ORDER);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
        return VM_FAULT_FALLBACK;
}

static vm_fault_t wp_huge_pud(struct vm_fault *vmf, pud_t orig_pud)
{
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) &&                        \
        defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD)
        struct vm_area_struct *vma = vmf->vma;
        vm_fault_t ret;

        /* No support for anonymous transparent PUD pages yet */
        if (vma_is_anonymous(vma))
                goto split;
        if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) {
                if (vma->vm_ops->huge_fault) {
                        ret = vma->vm_ops->huge_fault(vmf, PUD_ORDER);
                        if (!(ret & VM_FAULT_FALLBACK))
                                return ret;
                }
        }
split:
        /* COW or write-notify not handled on PUD level: split pud.*/
        __split_huge_pud(vma, vmf->pud, vmf->address);
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
        return VM_FAULT_FALLBACK;
}

/*
 * These routines also need to handle stuff like marking pages dirty
 * and/or accessed for architectures that don't do it in hardware (most
 * RISC architectures).  The early dirtying is also good on the i386.
 *
 * There is also a hook called "update_mmu_cache()" that architectures
 * with external mmu caches can use to update those (ie the Sparc or
 * PowerPC hashed page tables that act as extended TLBs).
 *
 * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
 * concurrent faults).
 *
 * The mmap_lock may have been released depending on flags and our return value.
 * See filemap_fault() and __folio_lock_or_retry().
 */
static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
{
        pte_t entry;

        if (unlikely(pmd_none(*vmf->pmd))) {
                /*
                 * Leave __pte_alloc() until later: because vm_ops->fault may
                 * want to allocate huge page, and if we expose page table
                 * for an instant, it will be difficult to retract from
                 * concurrent faults and from rmap lookups.
                 */
                vmf->pte = NULL;
                vmf->flags &= ~FAULT_FLAG_ORIG_PTE_VALID;
        } else {
                pmd_t dummy_pmdval;

                /*
                 * A regular pmd is established and it can't morph into a huge
                 * pmd by anon khugepaged, since that takes mmap_lock in write
                 * mode; but shmem or file collapse to THP could still morph
                 * it into a huge pmd: just retry later if so.
                 *
                 * Use the maywrite version to indicate that vmf->pte may be
                 * modified, but since we will use pte_same() to detect the
                 * change of the !pte_none() entry, there is no need to recheck
                 * the pmdval. Here we chooes to pass a dummy variable instead
                 * of NULL, which helps new user think about why this place is
                 * special.
                 */
                vmf->pte = pte_offset_map_rw_nolock(vmf->vma->vm_mm, vmf->pmd,
                                                    vmf->address, &dummy_pmdval,
                                                    &vmf->ptl);
                if (unlikely(!vmf->pte))
                        return 0;
                vmf->orig_pte = ptep_get_lockless(vmf->pte);
                vmf->flags |= FAULT_FLAG_ORIG_PTE_VALID;

                if (pte_none(vmf->orig_pte)) {
                        pte_unmap(vmf->pte);
                        vmf->pte = NULL;
                }
        }

        if (!vmf->pte)
                return do_pte_missing(vmf);

        if (!pte_present(vmf->orig_pte))
                return do_swap_page(vmf);

        if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
                return do_numa_page(vmf);

        spin_lock(vmf->ptl);
        entry = vmf->orig_pte;
        if (unlikely(!pte_same(ptep_get(vmf->pte), entry))) {
                update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
                goto unlock;
        }
        if (vmf->flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
                if (!pte_write(entry))
                        return do_wp_page(vmf);
                else if (likely(vmf->flags & FAULT_FLAG_WRITE))
                        entry = pte_mkdirty(entry);
        }
        entry = pte_mkyoung(entry);
        if (ptep_set_access_flags(vmf->vma, vmf->address, vmf->pte, entry,
                                vmf->flags & FAULT_FLAG_WRITE)) {
                update_mmu_cache_range(vmf, vmf->vma, vmf->address,
                                vmf->pte, 1);
        } else {
                /* Skip spurious TLB flush for retried page fault */
                if (vmf->flags & FAULT_FLAG_TRIED)
                        goto unlock;
                /*
                 * This is needed only for protection faults but the arch code
                 * is not yet telling us if this is a protection fault or not.
                 * This still avoids useless tlb flushes for .text page faults
                 * with threads.
                 */
                if (vmf->flags & FAULT_FLAG_WRITE)
                        flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
                                                     vmf->pte);
        }
unlock:
        pte_unmap_unlock(vmf->pte, vmf->ptl);
        return 0;
}

/*
 * On entry, we hold either the VMA lock or the mmap_lock
 * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
 * the result, the mmap_lock is not held on exit.  See filemap_fault()
 * and __folio_lock_or_retry().
 */
static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
                unsigned long address, unsigned int flags)
{
        struct vm_fault vmf = {
                .vma = vma,
                .address = address & PAGE_MASK,
                .real_address = address,
                .flags = flags,
                .pgoff = linear_page_index(vma, address),
                .gfp_mask = __get_fault_gfp_mask(vma),
        };
        struct mm_struct *mm = vma->vm_mm;
        unsigned long vm_flags = vma->vm_flags;
        pgd_t *pgd;
        p4d_t *p4d;
        vm_fault_t ret;

        pgd = pgd_offset(mm, address);
        p4d = p4d_alloc(mm, pgd, address);
        if (!p4d)
                return VM_FAULT_OOM;

        vmf.pud = pud_alloc(mm, p4d, address);
        if (!vmf.pud)
                return VM_FAULT_OOM;
retry_pud:
        if (pud_none(*vmf.pud) &&
            thp_vma_allowable_order(vma, vm_flags,
                                TVA_IN_PF | TVA_ENFORCE_SYSFS, PUD_ORDER)) {
                ret = create_huge_pud(&vmf);
                if (!(ret & VM_FAULT_FALLBACK))
                        return ret;
        } else {
                pud_t orig_pud = *vmf.pud;

                barrier();
                if (pud_trans_huge(orig_pud) || pud_devmap(orig_pud)) {

                        /*
                         * TODO once we support anonymous PUDs: NUMA case and
                         * FAULT_FLAG_UNSHARE handling.
                         */
                        if ((flags & FAULT_FLAG_WRITE) && !pud_write(orig_pud)) {
                                ret = wp_huge_pud(&vmf, orig_pud);
                                if (!(ret & VM_FAULT_FALLBACK))
                                        return ret;
                        } else {
                                huge_pud_set_accessed(&vmf, orig_pud);
                                return 0;
                        }
                }
        }

        vmf.pmd = pmd_alloc(mm, vmf.pud, address);
        if (!vmf.pmd)
                return VM_FAULT_OOM;

        /* Huge pud page fault raced with pmd_alloc? */
        if (pud_trans_unstable(vmf.pud))
                goto retry_pud;

        if (pmd_none(*vmf.pmd) &&
            thp_vma_allowable_order(vma, vm_flags,
                                TVA_IN_PF | TVA_ENFORCE_SYSFS, PMD_ORDER)) {
                ret = create_huge_pmd(&vmf);
                if (!(ret & VM_FAULT_FALLBACK))
                        return ret;
        } else {
                vmf.orig_pmd = pmdp_get_lockless(vmf.pmd);

                if (unlikely(is_swap_pmd(vmf.orig_pmd))) {
                        VM_BUG_ON(thp_migration_supported() &&
                                          !is_pmd_migration_entry(vmf.orig_pmd));
                        if (is_pmd_migration_entry(vmf.orig_pmd))
                                pmd_migration_entry_wait(mm, vmf.pmd);
                        return 0;
                }
                if (pmd_trans_huge(vmf.orig_pmd) || pmd_devmap(vmf.orig_pmd)) {
                        if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma))
                                return do_huge_pmd_numa_page(&vmf);

                        if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
                            !pmd_write(vmf.orig_pmd)) {
                                ret = wp_huge_pmd(&vmf);
                                if (!(ret & VM_FAULT_FALLBACK))
                                        return ret;
                        } else {
                                huge_pmd_set_accessed(&vmf);
                                return 0;
                        }
                }
        }

        return handle_pte_fault(&vmf);
}

/**
 * mm_account_fault - Do page fault accounting
 * @mm: mm from which memcg should be extracted. It can be NULL.
 * @regs: the pt_regs struct pointer.  When set to NULL, will skip accounting
 *        of perf event counters, but we'll still do the per-task accounting to
 *        the task who triggered this page fault.
 * @address: the faulted address.
 * @flags: the fault flags.
 * @ret: the fault retcode.
 *
 * This will take care of most of the page fault accounting.  Meanwhile, it
 * will also include the PERF_COUNT_SW_PAGE_FAULTS_[MAJ|MIN] perf counter
 * updates.  However, note that the handling of PERF_COUNT_SW_PAGE_FAULTS should
 * still be in per-arch page fault handlers at the entry of page fault.
 */
static inline void mm_account_fault(struct mm_struct *mm, struct pt_regs *regs,
                                    unsigned long address, unsigned int flags,
                                    vm_fault_t ret)
{
        bool major;

        /* Incomplete faults will be accounted upon completion. */
        if (ret & VM_FAULT_RETRY)
                return;

        /*
         * To preserve the behavior of older kernels, PGFAULT counters record
         * both successful and failed faults, as opposed to perf counters,
         * which ignore failed cases.
         */
        count_vm_event(PGFAULT);
        count_memcg_event_mm(mm, PGFAULT);

        /*
         * Do not account for unsuccessful faults (e.g. when the address wasn't
         * valid).  That includes arch_vma_access_permitted() failing before
         * reaching here. So this is not a "this many hardware page faults"
         * counter.  We should use the hw profiling for that.
         */
        if (ret & VM_FAULT_ERROR)
                return;

        /*
         * We define the fault as a major fault when the final successful fault
         * is VM_FAULT_MAJOR, or if it retried (which implies that we couldn't
         * handle it immediately previously).
         */
        major = (ret & VM_FAULT_MAJOR) || (flags & FAULT_FLAG_TRIED);

        if (major)
                current->maj_flt++;
        else
                current->min_flt++;

        /*
         * If the fault is done for GUP, regs will be NULL.  We only do the
         * accounting for the per thread fault counters who triggered the
         * fault, and we skip the perf event updates.
         */
        if (!regs)
                return;

        if (major)
                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs, address);
        else
                perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs, address);
}

#ifdef CONFIG_LRU_GEN
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
        /* the LRU algorithm only applies to accesses with recency */
        current->in_lru_fault = vma_has_recency(vma);
}

static void lru_gen_exit_fault(void)
{
        current->in_lru_fault = false;
}
#else
static void lru_gen_enter_fault(struct vm_area_struct *vma)
{
}

static void lru_gen_exit_fault(void)
{
}
#endif /* CONFIG_LRU_GEN */

static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
                                       unsigned int *flags)
{
        if (unlikely(*flags & FAULT_FLAG_UNSHARE)) {
                if (WARN_ON_ONCE(*flags & FAULT_FLAG_WRITE))
                        return VM_FAULT_SIGSEGV;
                /*
                 * FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
                 * just treat it like an ordinary read-fault otherwise.
                 */
                if (!is_cow_mapping(vma->vm_flags))
                        *flags &= ~FAULT_FLAG_UNSHARE;
        } else if (*flags & FAULT_FLAG_WRITE) {
                /* Write faults on read-only mappings are impossible ... */
                if (WARN_ON_ONCE(!(vma->vm_flags & VM_MAYWRITE)))
                        return VM_FAULT_SIGSEGV;
                /* ... and FOLL_FORCE only applies to COW mappings. */
                if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
                                 !is_cow_mapping(vma->vm_flags)))
                        return VM_FAULT_SIGSEGV;
        }
#ifdef CONFIG_PER_VMA_LOCK
        /*
         * Per-VMA locks can't be used with FAULT_FLAG_RETRY_NOWAIT because of
         * the assumption that lock is dropped on VM_FAULT_RETRY.
         */
        if (WARN_ON_ONCE((*flags &
                        (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)) ==
                        (FAULT_FLAG_VMA_LOCK | FAULT_FLAG_RETRY_NOWAIT)))
                return VM_FAULT_SIGSEGV;
#endif

        return 0;
}

/*
 * By the time we get here, we already hold either the VMA lock or the
 * mmap_lock (FAULT_FLAG_VMA_LOCK tells you which).
 *
 * The mmap_lock may have been released depending on flags and our
 * return value.  See filemap_fault() and __folio_lock_or_retry().
 */
vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
                           unsigned int flags, struct pt_regs *regs)
{
        /* If the fault handler drops the mmap_lock, vma may be freed */
        struct mm_struct *mm = vma->vm_mm;
        vm_fault_t ret;
        bool is_droppable;

        __set_current_state(TASK_RUNNING);

        ret = sanitize_fault_flags(vma, &flags);
        if (ret)
                goto out;

        if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
                                            flags & FAULT_FLAG_INSTRUCTION,
                                            flags & FAULT_FLAG_REMOTE)) {
                ret = VM_FAULT_SIGSEGV;
                goto out;
        }

        is_droppable = !!(vma->vm_flags & VM_DROPPABLE);

        /*
         * Enable the memcg OOM handling for faults triggered in user
         * space.  Kernel faults are handled more gracefully.
         */
        if (flags & FAULT_FLAG_USER)
                mem_cgroup_enter_user_fault();

        lru_gen_enter_fault(vma);

        if (unlikely(is_vm_hugetlb_page(vma)))
                ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
        else
                ret = __handle_mm_fault(vma, address, flags);

        /*
         * Warning: It is no longer safe to dereference vma-> after this point,
         * because mmap_lock might have been dropped by __handle_mm_fault(), so
         * vma might be destroyed from underneath us.
         */

        lru_gen_exit_fault();

        /* If the mapping is droppable, then errors due to OOM aren't fatal. */
        if (is_droppable)
                ret &= ~VM_FAULT_OOM;

        if (flags & FAULT_FLAG_USER) {
                mem_cgroup_exit_user_fault();
                /*
                 * The task may have entered a memcg OOM situation but
                 * if the allocation error was handled gracefully (no
                 * VM_FAULT_OOM), there is no need to kill anything.
                 * Just clean up the OOM state peacefully.
                 */
                if (task_in_memcg_oom(current) && !(ret & VM_FAULT_OOM))
                        mem_cgroup_oom_synchronize(false);
        }
out:
        mm_account_fault(mm, regs, address, flags, ret);

        return ret;
}
EXPORT_SYMBOL_GPL(handle_mm_fault);

#ifndef __PAGETABLE_P4D_FOLDED
/*
 * Allocate p4d page table.
 * We've already handled the fast-path in-line.
 */
int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
{
        p4d_t *new = p4d_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        spin_lock(&mm->page_table_lock);
        if (pgd_present(*pgd)) {        /* Another has populated it */
                p4d_free(mm, new);
        } else {
                smp_wmb(); /* See comment in pmd_install() */
                pgd_populate(mm, pgd, new);
        }
        spin_unlock(&mm->page_table_lock);
        return 0;
}
#endif /* __PAGETABLE_P4D_FOLDED */

#ifndef __PAGETABLE_PUD_FOLDED
/*
 * Allocate page upper directory.
 * We've already handled the fast-path in-line.
 */
int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address)
{
        pud_t *new = pud_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        spin_lock(&mm->page_table_lock);
        if (!p4d_present(*p4d)) {
                mm_inc_nr_puds(mm);
                smp_wmb(); /* See comment in pmd_install() */
                p4d_populate(mm, p4d, new);
        } else        /* Another has populated it */
                pud_free(mm, new);
        spin_unlock(&mm->page_table_lock);
        return 0;
}
#endif /* __PAGETABLE_PUD_FOLDED */

#ifndef __PAGETABLE_PMD_FOLDED
/*
 * Allocate page middle directory.
 * We've already handled the fast-path in-line.
 */
int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
        spinlock_t *ptl;
        pmd_t *new = pmd_alloc_one(mm, address);
        if (!new)
                return -ENOMEM;

        ptl = pud_lock(mm, pud);
        if (!pud_present(*pud)) {
                mm_inc_nr_pmds(mm);
                smp_wmb(); /* See comment in pmd_install() */
                pud_populate(mm, pud, new);
        } else {        /* Another has populated it */
                pmd_free(mm, new);
        }
        spin_unlock(ptl);
        return 0;
}
#endif /* __PAGETABLE_PMD_FOLDED */

static inline void pfnmap_args_setup(struct follow_pfnmap_args *args,
                                     spinlock_t *lock, pte_t *ptep,
                                     pgprot_t pgprot, unsigned long pfn_base,
                                     unsigned long addr_mask, bool writable,
                                     bool special)
{
        args->lock = lock;
        args->ptep = ptep;
        args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT);
        args->addr_mask = addr_mask;
        args->pgprot = pgprot;
        args->writable = writable;
        args->special = special;
}

static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma)
{
#ifdef CONFIG_LOCKDEP
        struct file *file = vma->vm_file;
        struct address_space *mapping = file ? file->f_mapping : NULL;

        if (mapping)
                lockdep_assert(lockdep_is_held(&mapping->i_mmap_rwsem) ||
                               lockdep_is_held(&vma->vm_mm->mmap_lock));
        else
                lockdep_assert(lockdep_is_held(&vma->vm_mm->mmap_lock));
#endif
}

/**
 * follow_pfnmap_start() - Look up a pfn mapping at a user virtual address
 * @args: Pointer to struct @follow_pfnmap_args
 *
 * The caller needs to setup args->vma and args->address to point to the
 * virtual address as the target of such lookup.  On a successful return,
 * the results will be put into other output fields.
 *
 * After the caller finished using the fields, the caller must invoke
 * another follow_pfnmap_end() to proper releases the locks and resources
 * of such look up request.
 *
 * During the start() and end() calls, the results in @args will be valid
 * as proper locks will be held.  After the end() is called, all the fields
 * in @follow_pfnmap_args will be invalid to be further accessed.  Further
 * use of such information after end() may require proper synchronizations
 * by the caller with page table updates, otherwise it can create a
 * security bug.
 *
 * If the PTE maps a refcounted page, callers are responsible to protect
 * against invalidation with MMU notifiers; otherwise access to the PFN at
 * a later point in time can trigger use-after-free.
 *
 * Only IO mappings and raw PFN mappings are allowed.  The mmap semaphore
 * should be taken for read, and the mmap semaphore cannot be released
 * before the end() is invoked.
 *
 * This function must not be used to modify PTE content.
 *
 * Return: zero on success, negative otherwise.
 */
int follow_pfnmap_start(struct follow_pfnmap_args *args)
{
        struct vm_area_struct *vma = args->vma;
        unsigned long address = args->address;
        struct mm_struct *mm = vma->vm_mm;
        spinlock_t *lock;
        pgd_t *pgdp;
        p4d_t *p4dp, p4d;
        pud_t *pudp, pud;
        pmd_t *pmdp, pmd;
        pte_t *ptep, pte;

        pfnmap_lockdep_assert(vma);

        if (unlikely(address < vma->vm_start || address >= vma->vm_end))
                goto out;

        if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
                goto out;
retry:
        pgdp = pgd_offset(mm, address);
        if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
                goto out;

        p4dp = p4d_offset(pgdp, address);
        p4d = READ_ONCE(*p4dp);
        if (p4d_none(p4d) || unlikely(p4d_bad(p4d)))
                goto out;

        pudp = pud_offset(p4dp, address);
        pud = READ_ONCE(*pudp);
        if (pud_none(pud))
                goto out;
        if (pud_leaf(pud)) {
                lock = pud_lock(mm, pudp);
                if (!unlikely(pud_leaf(pud))) {
                        spin_unlock(lock);
                        goto retry;
                }
                pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud),
                                  pud_pfn(pud), PUD_MASK, pud_write(pud),
                                  pud_special(pud));
                return 0;
        }

        pmdp = pmd_offset(pudp, address);
        pmd = pmdp_get_lockless(pmdp);
        if (pmd_leaf(pmd)) {
                lock = pmd_lock(mm, pmdp);
                if (!unlikely(pmd_leaf(pmd))) {
                        spin_unlock(lock);
                        goto retry;
                }
                pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd),
                                  pmd_pfn(pmd), PMD_MASK, pmd_write(pmd),
                                  pmd_special(pmd));
                return 0;
        }

        ptep = pte_offset_map_lock(mm, pmdp, address, &lock);
        if (!ptep)
                goto out;
        pte = ptep_get(ptep);
        if (!pte_present(pte))
                goto unlock;
        pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte),
                          pte_pfn(pte), PAGE_MASK, pte_write(pte),
                          pte_special(pte));
        return 0;
unlock:
        pte_unmap_unlock(ptep, lock);
out:
        return -EINVAL;
}
EXPORT_SYMBOL_GPL(follow_pfnmap_start);

/**
 * follow_pfnmap_end(): End a follow_pfnmap_start() process
 * @args: Pointer to struct @follow_pfnmap_args
 *
 * Must be used in pair of follow_pfnmap_start().  See the start() function
 * above for more information.
 */
void follow_pfnmap_end(struct follow_pfnmap_args *args)
{
        if (args->lock)
                spin_unlock(args->lock);
        if (args->ptep)
                pte_unmap(args->ptep);
}
EXPORT_SYMBOL_GPL(follow_pfnmap_end);

#ifdef CONFIG_HAVE_IOREMAP_PROT
/**
 * generic_access_phys - generic implementation for iomem mmap access
 * @vma: the vma to access
 * @addr: userspace address, not relative offset within @vma
 * @buf: buffer to read/write
 * @len: length of transfer
 * @write: set to FOLL_WRITE when writing, otherwise reading
 *
 * This is a generic implementation for &vm_operations_struct.access for an
 * iomem mapping. This callback is used by access_process_vm() when the @vma is
 * not page based.
 */
int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
                        void *buf, int len, int write)
{
        resource_size_t phys_addr;
        pgprot_t prot = __pgprot(0);
        void __iomem *maddr;
        int offset = offset_in_page(addr);
        int ret = -EINVAL;
        bool writable;
        struct follow_pfnmap_args args = { .vma = vma, .address = addr };

retry:
        if (follow_pfnmap_start(&args))
                return -EINVAL;
        prot = args.pgprot;
        phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT;
        writable = args.writable;
        follow_pfnmap_end(&args);

        if ((write & FOLL_WRITE) && !writable)
                return -EINVAL;

        maddr = ioremap_prot(phys_addr, PAGE_ALIGN(len + offset), prot);
        if (!maddr)
                return -ENOMEM;

        if (follow_pfnmap_start(&args))
                goto out_unmap;

        if ((pgprot_val(prot) != pgprot_val(args.pgprot)) ||
            (phys_addr != (args.pfn << PAGE_SHIFT)) ||
            (writable != args.writable)) {
                follow_pfnmap_end(&args);
                iounmap(maddr);
                goto retry;
        }

        if (write)
                memcpy_toio(maddr + offset, buf, len);
        else
                memcpy_fromio(buf, maddr + offset, len);
        ret = len;
        follow_pfnmap_end(&args);
out_unmap:
        iounmap(maddr);

        return ret;
}
EXPORT_SYMBOL_GPL(generic_access_phys);
#endif

/*
 * Access another process' address space as given in mm.
 */
static int __access_remote_vm(struct mm_struct *mm, unsigned long addr,
                              void *buf, int len, unsigned int gup_flags)
{
        void *old_buf = buf;
        int write = gup_flags & FOLL_WRITE;

        if (mmap_read_lock_killable(mm))
                return 0;

        /* Untag the address before looking up the VMA */
        addr = untagged_addr_remote(mm, addr);

        /* Avoid triggering the temporary warning in __get_user_pages */
        if (!vma_lookup(mm, addr) && !expand_stack(mm, addr))
                return 0;

        /* ignore errors, just check how much was successfully transferred */
        while (len) {
                int bytes, offset;
                void *maddr;
                struct vm_area_struct *vma = NULL;
                struct page *page = get_user_page_vma_remote(mm, addr,
                                                             gup_flags, &vma);

                if (IS_ERR(page)) {
                        /* We might need to expand the stack to access it */
                        vma = vma_lookup(mm, addr);
                        if (!vma) {
                                vma = expand_stack(mm, addr);

                                /* mmap_lock was dropped on failure */
                                if (!vma)
                                        return buf - old_buf;

                                /* Try again if stack expansion worked */
                                continue;
                        }

                        /*
                         * Check if this is a VM_IO | VM_PFNMAP VMA, which
                         * we can access using slightly different code.
                         */
                        bytes = 0;
#ifdef CONFIG_HAVE_IOREMAP_PROT
                        if (vma->vm_ops && vma->vm_ops->access)
                                bytes = vma->vm_ops->access(vma, addr, buf,
                                                            len, write);
#endif
                        if (bytes <= 0)
                                break;
                } else {
                        bytes = len;
                        offset = addr & (PAGE_SIZE-1);
                        if (bytes > PAGE_SIZE-offset)
                                bytes = PAGE_SIZE-offset;

                        maddr = kmap_local_page(page);
                        if (write) {
                                copy_to_user_page(vma, page, addr,
                                                  maddr + offset, buf, bytes);
                                set_page_dirty_lock(page);
                        } else {
                                copy_from_user_page(vma, page, addr,
                                                    buf, maddr + offset, bytes);
                        }
                        unmap_and_put_page(page, maddr);
                }
                len -= bytes;
                buf += bytes;
                addr += bytes;
        }
        mmap_read_unlock(mm);

        return buf - old_buf;
}

/**
 * access_remote_vm - access another process' address space
 * @mm:                the mm_struct of the target address space
 * @addr:        start address to access
 * @buf:        source or destination buffer
 * @len:        number of bytes to transfer
 * @gup_flags:        flags modifying lookup behaviour
 *
 * The caller must hold a reference on @mm.
 *
 * Return: number of bytes copied from source to destination.
 */
int access_remote_vm(struct mm_struct *mm, unsigned long addr,
                void *buf, int len, unsigned int gup_flags)
{
        return __access_remote_vm(mm, addr, buf, len, gup_flags);
}

/*
 * Access another process' address space.
 * Source/target buffer must be kernel space,
 * Do not walk the page table directly, use get_user_pages
 */
int access_process_vm(struct task_struct *tsk, unsigned long addr,
                void *buf, int len, unsigned int gup_flags)
{
        struct mm_struct *mm;
        int ret;

        mm = get_task_mm(tsk);
        if (!mm)
                return 0;

        ret = __access_remote_vm(mm, addr, buf, len, gup_flags);

        mmput(mm);

        return ret;
}
EXPORT_SYMBOL_GPL(access_process_vm);

#ifdef CONFIG_BPF_SYSCALL
/*
 * Copy a string from another process's address space as given in mm.
 * If there is any error return -EFAULT.
 */
static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
                                void *buf, int len, unsigned int gup_flags)
{
        void *old_buf = buf;
        int err = 0;

        *(char *)buf = '\0';

        if (mmap_read_lock_killable(mm))
                return -EFAULT;

        addr = untagged_addr_remote(mm, addr);

        /* Avoid triggering the temporary warning in __get_user_pages */
        if (!vma_lookup(mm, addr)) {
                err = -EFAULT;
                goto out;
        }

        while (len) {
                int bytes, offset, retval;
                void *maddr;
                struct page *page;
                struct vm_area_struct *vma = NULL;

                page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
                if (IS_ERR(page)) {
                        /*
                         * Treat as a total failure for now until we decide how
                         * to handle the CONFIG_HAVE_IOREMAP_PROT case and
                         * stack expansion.
                         */
                        *(char *)buf = '\0';
                        err = -EFAULT;
                        goto out;
                }

                bytes = len;
                offset = addr & (PAGE_SIZE - 1);
                if (bytes > PAGE_SIZE - offset)
                        bytes = PAGE_SIZE - offset;

                maddr = kmap_local_page(page);
                retval = strscpy(buf, maddr + offset, bytes);
                if (retval >= 0) {
                        /* Found the end of the string */
                        buf += retval;
                        unmap_and_put_page(page, maddr);
                        break;
                }

                buf += bytes - 1;
                /*
                 * Because strscpy always NUL terminates we need to
                 * copy the last byte in the page if we are going to
                 * load more pages
                 */
                if (bytes != len) {
                        addr += bytes - 1;
                        copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1);
                        buf += 1;
                        addr += 1;
                }
                len -= bytes;

                unmap_and_put_page(page, maddr);
        }

out:
        mmap_read_unlock(mm);
        if (err)
                return err;
        return buf - old_buf;
}

/**
 * copy_remote_vm_str - copy a string from another process's address space.
 * @tsk:        the task of the target address space
 * @addr:        start address to read from
 * @buf:        destination buffer
 * @len:        number of bytes to copy
 * @gup_flags:        flags modifying lookup behaviour
 *
 * The caller must hold a reference on @mm.
 *
 * Return: number of bytes copied from @addr (source) to @buf (destination);
 * not including the trailing NUL. Always guaranteed to leave NUL-terminated
 * buffer. On any error, return -EFAULT.
 */
int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
                       void *buf, int len, unsigned int gup_flags)
{
        struct mm_struct *mm;
        int ret;

        if (unlikely(len == 0))
                return 0;

        mm = get_task_mm(tsk);
        if (!mm) {
                *(char *)buf = '\0';
                return -EFAULT;
        }

        ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);

        mmput(mm);

        return ret;
}
EXPORT_SYMBOL_GPL(copy_remote_vm_str);
#endif /* CONFIG_BPF_SYSCALL */

/*
 * Print the name of a VMA.
 */
void print_vma_addr(char *prefix, unsigned long ip)
{
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;

        /*
         * we might be running from an atomic context so we cannot sleep
         */
        if (!mmap_read_trylock(mm))
                return;

        vma = vma_lookup(mm, ip);
        if (vma && vma->vm_file) {
                struct file *f = vma->vm_file;
                ip -= vma->vm_start;
                ip += vma->vm_pgoff << PAGE_SHIFT;
                printk("%s%pD[%lx,%lx+%lx]", prefix, f, ip,
                                vma->vm_start,
                                vma->vm_end - vma->vm_start);
        }
        mmap_read_unlock(mm);
}

#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)
void __might_fault(const char *file, int line)
{
        if (pagefault_disabled())
                return;
        __might_sleep(file, line);
        if (current->mm)
                might_lock_read(&current->mm->mmap_lock);
}
EXPORT_SYMBOL(__might_fault);
#endif

#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
/*
 * Process all subpages of the specified huge page with the specified
 * operation.  The target subpage will be processed last to keep its
 * cache lines hot.
 */
static inline int process_huge_page(
        unsigned long addr_hint, unsigned int nr_pages,
        int (*process_subpage)(unsigned long addr, int idx, void *arg),
        void *arg)
{
        int i, n, base, l, ret;
        unsigned long addr = addr_hint &
                ~(((unsigned long)nr_pages << PAGE_SHIFT) - 1);

        /* Process target subpage last to keep its cache lines hot */
        might_sleep();
        n = (addr_hint - addr) / PAGE_SIZE;
        if (2 * n <= nr_pages) {
                /* If target subpage in first half of huge page */
                base = 0;
                l = n;
                /* Process subpages at the end of huge page */
                for (i = nr_pages - 1; i >= 2 * n; i--) {
                        cond_resched();
                        ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
                        if (ret)
                                return ret;
                }
        } else {
                /* If target subpage in second half of huge page */
                base = nr_pages - 2 * (nr_pages - n);
                l = nr_pages - n;
                /* Process subpages at the begin of huge page */
                for (i = 0; i < base; i++) {
                        cond_resched();
                        ret = process_subpage(addr + i * PAGE_SIZE, i, arg);
                        if (ret)
                                return ret;
                }
        }
        /*
         * Process remaining subpages in left-right-left-right pattern
         * towards the target subpage
         */
        for (i = 0; i < l; i++) {
                int left_idx = base + i;
                int right_idx = base + 2 * l - 1 - i;

                cond_resched();
                ret = process_subpage(addr + left_idx * PAGE_SIZE, left_idx, arg);
                if (ret)
                        return ret;
                cond_resched();
                ret = process_subpage(addr + right_idx * PAGE_SIZE, right_idx, arg);
                if (ret)
                        return ret;
        }
        return 0;
}

static void clear_gigantic_page(struct folio *folio, unsigned long addr_hint,
                                unsigned int nr_pages)
{
        unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(folio));
        int i;

        might_sleep();
        for (i = 0; i < nr_pages; i++) {
                cond_resched();
                clear_user_highpage(folio_page(folio, i), addr + i * PAGE_SIZE);
        }
}

static int clear_subpage(unsigned long addr, int idx, void *arg)
{
        struct folio *folio = arg;

        clear_user_highpage(folio_page(folio, idx), addr);
        return 0;
}

/**
 * folio_zero_user - Zero a folio which will be mapped to userspace.
 * @folio: The folio to zero.
 * @addr_hint: The address will be accessed or the base address if uncelar.
 */
void folio_zero_user(struct folio *folio, unsigned long addr_hint)
{
        unsigned int nr_pages = folio_nr_pages(folio);

        if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
                clear_gigantic_page(folio, addr_hint, nr_pages);
        else
                process_huge_page(addr_hint, nr_pages, clear_subpage, folio);
}

static int copy_user_gigantic_page(struct folio *dst, struct folio *src,
                                   unsigned long addr_hint,
                                   struct vm_area_struct *vma,
                                   unsigned int nr_pages)
{
        unsigned long addr = ALIGN_DOWN(addr_hint, folio_size(dst));
        struct page *dst_page;
        struct page *src_page;
        int i;

        for (i = 0; i < nr_pages; i++) {
                dst_page = folio_page(dst, i);
                src_page = folio_page(src, i);

                cond_resched();
                if (copy_mc_user_highpage(dst_page, src_page,
                                          addr + i*PAGE_SIZE, vma))
                        return -EHWPOISON;
        }
        return 0;
}

struct copy_subpage_arg {
        struct folio *dst;
        struct folio *src;
        struct vm_area_struct *vma;
};

static int copy_subpage(unsigned long addr, int idx, void *arg)
{
        struct copy_subpage_arg *copy_arg = arg;
        struct page *dst = folio_page(copy_arg->dst, idx);
        struct page *src = folio_page(copy_arg->src, idx);

        if (copy_mc_user_highpage(dst, src, addr, copy_arg->vma))
                return -EHWPOISON;
        return 0;
}

int copy_user_large_folio(struct folio *dst, struct folio *src,
                          unsigned long addr_hint, struct vm_area_struct *vma)
{
        unsigned int nr_pages = folio_nr_pages(dst);
        struct copy_subpage_arg arg = {
                .dst = dst,
                .src = src,
                .vma = vma,
        };

        if (unlikely(nr_pages > MAX_ORDER_NR_PAGES))
                return copy_user_gigantic_page(dst, src, addr_hint, vma, nr_pages);

        return process_huge_page(addr_hint, nr_pages, copy_subpage, &arg);
}

long copy_folio_from_user(struct folio *dst_folio,
                           const void __user *usr_src,
                           bool allow_pagefault)
{
        void *kaddr;
        unsigned long i, rc = 0;
        unsigned int nr_pages = folio_nr_pages(dst_folio);
        unsigned long ret_val = nr_pages * PAGE_SIZE;
        struct page *subpage;

        for (i = 0; i < nr_pages; i++) {
                subpage = folio_page(dst_folio, i);
                kaddr = kmap_local_page(subpage);
                if (!allow_pagefault)
                        pagefault_disable();
                rc = copy_from_user(kaddr, usr_src + i * PAGE_SIZE, PAGE_SIZE);
                if (!allow_pagefault)
                        pagefault_enable();
                kunmap_local(kaddr);

                ret_val -= (PAGE_SIZE - rc);
                if (rc)
                        break;

                flush_dcache_page(subpage);

                cond_resched();
        }
        return ret_val;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */

#if defined(CONFIG_SPLIT_PTE_PTLOCKS) && ALLOC_SPLIT_PTLOCKS

static struct kmem_cache *page_ptl_cachep;

void __init ptlock_cache_init(void)
{
        page_ptl_cachep = kmem_cache_create("page->ptl", sizeof(spinlock_t), 0,
                        SLAB_PANIC, NULL);
}

bool ptlock_alloc(struct ptdesc *ptdesc)
{
        spinlock_t *ptl;

        ptl = kmem_cache_alloc(page_ptl_cachep, GFP_KERNEL);
        if (!ptl)
                return false;
        ptdesc->ptl = ptl;
        return true;
}

void ptlock_free(struct ptdesc *ptdesc)
{
        if (ptdesc->ptl)
                kmem_cache_free(page_ptl_cachep, ptdesc->ptl);
}
#endif

void vma_pgtable_walk_begin(struct vm_area_struct *vma)
{
        if (is_vm_hugetlb_page(vma))
                hugetlb_vma_lock_read(vma);
}

void vma_pgtable_walk_end(struct vm_area_struct *vma)
{
        if (is_vm_hugetlb_page(vma))
                hugetlb_vma_unlock_read(vma);
}














































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    5 



    2 





    4 
    2 


































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
// SPDX-License-Identifier: GPL-2.0
/* Generic nexthop implementation
 *
 * Copyright (c) 2017-19 Cumulus Networks
 * Copyright (c) 2017-19 David Ahern <dsa@cumulusnetworks.com>
 */

#include <linux/nexthop.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/arp.h>
#include <net/ipv6_stubs.h>
#include <net/lwtunnel.h>
#include <net/ndisc.h>
#include <net/nexthop.h>
#include <net/route.h>
#include <net/sock.h>

#define NH_RES_DEFAULT_IDLE_TIMER        (120 * HZ)
#define NH_RES_DEFAULT_UNBALANCED_TIMER        0        /* No forced rebalancing. */

static void remove_nexthop(struct net *net, struct nexthop *nh,
                           struct nl_info *nlinfo);

#define NH_DEV_HASHBITS  8
#define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)

#define NHA_OP_FLAGS_DUMP_ALL (NHA_OP_FLAG_DUMP_STATS |                \
                               NHA_OP_FLAG_DUMP_HW_STATS)

static const struct nla_policy rtm_nh_policy_new[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_GROUP]                = { .type = NLA_BINARY },
        [NHA_GROUP_TYPE]        = { .type = NLA_U16 },
        [NHA_BLACKHOLE]                = { .type = NLA_FLAG },
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_GATEWAY]                = { .type = NLA_BINARY },
        [NHA_ENCAP_TYPE]        = { .type = NLA_U16 },
        [NHA_ENCAP]                = { .type = NLA_NESTED },
        [NHA_FDB]                = { .type = NLA_FLAG },
        [NHA_RES_GROUP]                = { .type = NLA_NESTED },
        [NHA_HW_STATS_ENABLE]        = NLA_POLICY_MAX(NLA_U32, true),
};

static const struct nla_policy rtm_nh_policy_get[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_OP_FLAGS]                = NLA_POLICY_MASK(NLA_U32,
                                                  NHA_OP_FLAGS_DUMP_ALL),
};

static const struct nla_policy rtm_nh_policy_del[] = {
        [NHA_ID]                = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_dump[] = {
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_GROUPS]                = { .type = NLA_FLAG },
        [NHA_MASTER]                = { .type = NLA_U32 },
        [NHA_FDB]                = { .type = NLA_FLAG },
        [NHA_OP_FLAGS]                = NLA_POLICY_MASK(NLA_U32,
                                                  NHA_OP_FLAGS_DUMP_ALL),
};

static const struct nla_policy rtm_nh_res_policy_new[] = {
        [NHA_RES_GROUP_BUCKETS]                        = { .type = NLA_U16 },
        [NHA_RES_GROUP_IDLE_TIMER]                = { .type = NLA_U32 },
        [NHA_RES_GROUP_UNBALANCED_TIMER]        = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_dump_bucket[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_OIF]                = { .type = NLA_U32 },
        [NHA_MASTER]                = { .type = NLA_U32 },
        [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
};

static const struct nla_policy rtm_nh_res_bucket_policy_dump[] = {
        [NHA_RES_BUCKET_NH_ID]        = { .type = NLA_U32 },
};

static const struct nla_policy rtm_nh_policy_get_bucket[] = {
        [NHA_ID]                = { .type = NLA_U32 },
        [NHA_RES_BUCKET]        = { .type = NLA_NESTED },
};

static const struct nla_policy rtm_nh_res_bucket_policy_get[] = {
        [NHA_RES_BUCKET_INDEX]        = { .type = NLA_U16 },
};

static bool nexthop_notifiers_is_empty(struct net *net)
{
        return !net->nexthop.notifier_chain.head;
}

static void
__nh_notifier_single_info_init(struct nh_notifier_single_info *nh_info,
                               const struct nh_info *nhi)
{
        nh_info->dev = nhi->fib_nhc.nhc_dev;
        nh_info->gw_family = nhi->fib_nhc.nhc_gw_family;
        if (nh_info->gw_family == AF_INET)
                nh_info->ipv4 = nhi->fib_nhc.nhc_gw.ipv4;
        else if (nh_info->gw_family == AF_INET6)
                nh_info->ipv6 = nhi->fib_nhc.nhc_gw.ipv6;

        nh_info->id = nhi->nh_parent->id;
        nh_info->is_reject = nhi->reject_nh;
        nh_info->is_fdb = nhi->fdb_nh;
        nh_info->has_encap = !!nhi->fib_nhc.nhc_lwtstate;
}

static int nh_notifier_single_info_init(struct nh_notifier_info *info,
                                        const struct nexthop *nh)
{
        struct nh_info *nhi = rtnl_dereference(nh->nh_info);

        info->type = NH_NOTIFIER_INFO_TYPE_SINGLE;
        info->nh = kzalloc(sizeof(*info->nh), GFP_KERNEL);
        if (!info->nh)
                return -ENOMEM;

        __nh_notifier_single_info_init(info->nh, nhi);

        return 0;
}

static void nh_notifier_single_info_fini(struct nh_notifier_info *info)
{
        kfree(info->nh);
}

static int nh_notifier_mpath_info_init(struct nh_notifier_info *info,
                                       struct nh_group *nhg)
{
        u16 num_nh = nhg->num_nh;
        int i;

        info->type = NH_NOTIFIER_INFO_TYPE_GRP;
        info->nh_grp = kzalloc(struct_size(info->nh_grp, nh_entries, num_nh),
                               GFP_KERNEL);
        if (!info->nh_grp)
                return -ENOMEM;

        info->nh_grp->num_nh = num_nh;
        info->nh_grp->is_fdb = nhg->fdb_nh;
        info->nh_grp->hw_stats = nhg->hw_stats;

        for (i = 0; i < num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                struct nh_info *nhi;

                nhi = rtnl_dereference(nhge->nh->nh_info);
                info->nh_grp->nh_entries[i].weight = nhge->weight;
                __nh_notifier_single_info_init(&info->nh_grp->nh_entries[i].nh,
                                               nhi);
        }

        return 0;
}

static int nh_notifier_res_table_info_init(struct nh_notifier_info *info,
                                           struct nh_group *nhg)
{
        struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
        u16 num_nh_buckets = res_table->num_nh_buckets;
        unsigned long size;
        u16 i;

        info->type = NH_NOTIFIER_INFO_TYPE_RES_TABLE;
        size = struct_size(info->nh_res_table, nhs, num_nh_buckets);
        info->nh_res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO |
                                       __GFP_NOWARN);
        if (!info->nh_res_table)
                return -ENOMEM;

        info->nh_res_table->num_nh_buckets = num_nh_buckets;
        info->nh_res_table->hw_stats = nhg->hw_stats;

        for (i = 0; i < num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;
                struct nh_info *nhi;

                nhge = rtnl_dereference(bucket->nh_entry);
                nhi = rtnl_dereference(nhge->nh->nh_info);
                __nh_notifier_single_info_init(&info->nh_res_table->nhs[i],
                                               nhi);
        }

        return 0;
}

static int nh_notifier_grp_info_init(struct nh_notifier_info *info,
                                     const struct nexthop *nh)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

        if (nhg->hash_threshold)
                return nh_notifier_mpath_info_init(info, nhg);
        else if (nhg->resilient)
                return nh_notifier_res_table_info_init(info, nhg);
        return -EINVAL;
}

static void nh_notifier_grp_info_fini(struct nh_notifier_info *info,
                                      const struct nexthop *nh)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

        if (nhg->hash_threshold)
                kfree(info->nh_grp);
        else if (nhg->resilient)
                vfree(info->nh_res_table);
}

static int nh_notifier_info_init(struct nh_notifier_info *info,
                                 const struct nexthop *nh)
{
        info->id = nh->id;

        if (nh->is_group)
                return nh_notifier_grp_info_init(info, nh);
        else
                return nh_notifier_single_info_init(info, nh);
}

static void nh_notifier_info_fini(struct nh_notifier_info *info,
                                  const struct nexthop *nh)
{
        if (nh->is_group)
                nh_notifier_grp_info_fini(info, nh);
        else
                nh_notifier_single_info_fini(info);
}

static int call_nexthop_notifiers(struct net *net,
                                  enum nexthop_event_type event_type,
                                  struct nexthop *nh,
                                  struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
        };
        int err;

        ASSERT_RTNL();

        if (nexthop_notifiers_is_empty(net))
                return 0;

        err = nh_notifier_info_init(&info, nh);
        if (err) {
                NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
                return err;
        }

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           event_type, &info);
        nh_notifier_info_fini(&info, nh);

        return notifier_to_errno(err);
}

static int
nh_notifier_res_bucket_idle_timer_get(const struct nh_notifier_info *info,
                                      bool force, unsigned int *p_idle_timer_ms)
{
        struct nh_res_table *res_table;
        struct nh_group *nhg;
        struct nexthop *nh;
        int err = 0;

        /* When 'force' is false, nexthop bucket replacement is performed
         * because the bucket was deemed to be idle. In this case, capable
         * listeners can choose to perform an atomic replacement: The bucket is
         * only replaced if it is inactive. However, if the idle timer interval
         * is smaller than the interval in which a listener is querying
         * buckets' activity from the device, then atomic replacement should
         * not be tried. Pass the idle timer value to listeners, so that they
         * could determine which type of replacement to perform.
         */
        if (force) {
                *p_idle_timer_ms = 0;
                return 0;
        }

        rcu_read_lock();

        nh = nexthop_find_by_id(info->net, info->id);
        if (!nh) {
                err = -EINVAL;
                goto out;
        }

        nhg = rcu_dereference(nh->nh_grp);
        res_table = rcu_dereference(nhg->res_table);
        *p_idle_timer_ms = jiffies_to_msecs(res_table->idle_timer);

out:
        rcu_read_unlock();

        return err;
}

static int nh_notifier_res_bucket_info_init(struct nh_notifier_info *info,
                                            u16 bucket_index, bool force,
                                            struct nh_info *oldi,
                                            struct nh_info *newi)
{
        unsigned int idle_timer_ms;
        int err;

        err = nh_notifier_res_bucket_idle_timer_get(info, force,
                                                    &idle_timer_ms);
        if (err)
                return err;

        info->type = NH_NOTIFIER_INFO_TYPE_RES_BUCKET;
        info->nh_res_bucket = kzalloc(sizeof(*info->nh_res_bucket),
                                      GFP_KERNEL);
        if (!info->nh_res_bucket)
                return -ENOMEM;

        info->nh_res_bucket->bucket_index = bucket_index;
        info->nh_res_bucket->idle_timer_ms = idle_timer_ms;
        info->nh_res_bucket->force = force;
        __nh_notifier_single_info_init(&info->nh_res_bucket->old_nh, oldi);
        __nh_notifier_single_info_init(&info->nh_res_bucket->new_nh, newi);
        return 0;
}

static void nh_notifier_res_bucket_info_fini(struct nh_notifier_info *info)
{
        kfree(info->nh_res_bucket);
}

static int __call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
                                               u16 bucket_index, bool force,
                                               struct nh_info *oldi,
                                               struct nh_info *newi,
                                               struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
                .id = nhg_id,
        };
        int err;

        if (nexthop_notifiers_is_empty(net))
                return 0;

        err = nh_notifier_res_bucket_info_init(&info, bucket_index, force,
                                               oldi, newi);
        if (err)
                return err;

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_BUCKET_REPLACE, &info);
        nh_notifier_res_bucket_info_fini(&info);

        return notifier_to_errno(err);
}

/* There are three users of RES_TABLE, and NHs etc. referenced from there:
 *
 * 1) a collection of callbacks for NH maintenance. This operates under
 *    RTNL,
 * 2) the delayed work that gradually balances the resilient table,
 * 3) and nexthop_select_path(), operating under RCU.
 *
 * Both the delayed work and the RTNL block are writers, and need to
 * maintain mutual exclusion. Since there are only two and well-known
 * writers for each table, the RTNL code can make sure it has exclusive
 * access thus:
 *
 * - Have the DW operate without locking;
 * - synchronously cancel the DW;
 * - do the writing;
 * - if the write was not actually a delete, call upkeep, which schedules
 *   DW again if necessary.
 *
 * The functions that are always called from the RTNL context use
 * rtnl_dereference(). The functions that can also be called from the DW do
 * a raw dereference and rely on the above mutual exclusion scheme.
 */
#define nh_res_dereference(p) (rcu_dereference_raw(p))

static int call_nexthop_res_bucket_notifiers(struct net *net, u32 nhg_id,
                                             u16 bucket_index, bool force,
                                             struct nexthop *old_nh,
                                             struct nexthop *new_nh,
                                             struct netlink_ext_ack *extack)
{
        struct nh_info *oldi = nh_res_dereference(old_nh->nh_info);
        struct nh_info *newi = nh_res_dereference(new_nh->nh_info);

        return __call_nexthop_res_bucket_notifiers(net, nhg_id, bucket_index,
                                                   force, oldi, newi, extack);
}

static int call_nexthop_res_table_notifiers(struct net *net, struct nexthop *nh,
                                            struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
                .id = nh->id,
        };
        struct nh_group *nhg;
        int err;

        ASSERT_RTNL();

        if (nexthop_notifiers_is_empty(net))
                return 0;

        /* At this point, the nexthop buckets are still not populated. Only
         * emit a notification with the logical nexthops, so that a listener
         * could potentially veto it in case of unsupported configuration.
         */
        nhg = rtnl_dereference(nh->nh_grp);
        err = nh_notifier_mpath_info_init(&info, nhg);
        if (err) {
                NL_SET_ERR_MSG(extack, "Failed to initialize nexthop notifier info");
                return err;
        }

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_RES_TABLE_PRE_REPLACE,
                                           &info);
        kfree(info.nh_grp);

        return notifier_to_errno(err);
}

static int call_nexthop_notifier(struct notifier_block *nb, struct net *net,
                                 enum nexthop_event_type event_type,
                                 struct nexthop *nh,
                                 struct netlink_ext_ack *extack)
{
        struct nh_notifier_info info = {
                .net = net,
                .extack = extack,
        };
        int err;

        err = nh_notifier_info_init(&info, nh);
        if (err)
                return err;

        err = nb->notifier_call(nb, event_type, &info);
        nh_notifier_info_fini(&info, nh);

        return notifier_to_errno(err);
}

static unsigned int nh_dev_hashfn(unsigned int val)
{
        unsigned int mask = NH_DEV_HASHSIZE - 1;

        return (val ^
                (val >> NH_DEV_HASHBITS) ^
                (val >> (NH_DEV_HASHBITS * 2))) & mask;
}

static void nexthop_devhash_add(struct net *net, struct nh_info *nhi)
{
        struct net_device *dev = nhi->fib_nhc.nhc_dev;
        struct hlist_head *head;
        unsigned int hash;

        WARN_ON(!dev);

        hash = nh_dev_hashfn(dev->ifindex);
        head = &net->nexthop.devhash[hash];
        hlist_add_head(&nhi->dev_hash, head);
}

static void nexthop_free_group(struct nexthop *nh)
{
        struct nh_group *nhg;
        int i;

        nhg = rcu_dereference_raw(nh->nh_grp);
        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                WARN_ON(!list_empty(&nhge->nh_list));
                free_percpu(nhge->stats);
                nexthop_put(nhge->nh);
        }

        WARN_ON(nhg->spare == nhg);

        if (nhg->resilient)
                vfree(rcu_dereference_raw(nhg->res_table));

        kfree(nhg->spare);
        kfree(nhg);
}

static void nexthop_free_single(struct nexthop *nh)
{
        struct nh_info *nhi;

        nhi = rcu_dereference_raw(nh->nh_info);
        switch (nhi->family) {
        case AF_INET:
                fib_nh_release(nh->net, &nhi->fib_nh);
                break;
        case AF_INET6:
                ipv6_stub->fib6_nh_release(&nhi->fib6_nh);
                break;
        }
        kfree(nhi);
}

void nexthop_free_rcu(struct rcu_head *head)
{
        struct nexthop *nh = container_of(head, struct nexthop, rcu);

        if (nh->is_group)
                nexthop_free_group(nh);
        else
                nexthop_free_single(nh);

        kfree(nh);
}
EXPORT_SYMBOL_GPL(nexthop_free_rcu);

static struct nexthop *nexthop_alloc(void)
{
        struct nexthop *nh;

        nh = kzalloc(sizeof(struct nexthop), GFP_KERNEL);
        if (nh) {
                INIT_LIST_HEAD(&nh->fi_list);
                INIT_LIST_HEAD(&nh->f6i_list);
                INIT_LIST_HEAD(&nh->grp_list);
                INIT_LIST_HEAD(&nh->fdb_list);
                spin_lock_init(&nh->lock);
        }
        return nh;
}

static struct nh_group *nexthop_grp_alloc(u16 num_nh)
{
        struct nh_group *nhg;

        nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);
        if (nhg)
                nhg->num_nh = num_nh;

        return nhg;
}

static void nh_res_table_upkeep_dw(struct work_struct *work);

static struct nh_res_table *
nexthop_res_table_alloc(struct net *net, u32 nhg_id, struct nh_config *cfg)
{
        const u16 num_nh_buckets = cfg->nh_grp_res_num_buckets;
        struct nh_res_table *res_table;
        unsigned long size;

        size = struct_size(res_table, nh_buckets, num_nh_buckets);
        res_table = __vmalloc(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN);
        if (!res_table)
                return NULL;

        res_table->net = net;
        res_table->nhg_id = nhg_id;
        INIT_DELAYED_WORK(&res_table->upkeep_dw, &nh_res_table_upkeep_dw);
        INIT_LIST_HEAD(&res_table->uw_nh_entries);
        res_table->idle_timer = cfg->nh_grp_res_idle_timer;
        res_table->unbalanced_timer = cfg->nh_grp_res_unbalanced_timer;
        res_table->num_nh_buckets = num_nh_buckets;
        return res_table;
}

static void nh_base_seq_inc(struct net *net)
{
        while (++net->nexthop.seq == 0)
                ;
}

/* no reference taken; rcu lock or rtnl must be held */
struct nexthop *nexthop_find_by_id(struct net *net, u32 id)
{
        struct rb_node **pp, *parent = NULL, *next;

        pp = &net->nexthop.rb_root.rb_node;
        while (1) {
                struct nexthop *nh;

                next = rcu_dereference_raw(*pp);
                if (!next)
                        break;
                parent = next;

                nh = rb_entry(parent, struct nexthop, rb_node);
                if (id < nh->id)
                        pp = &next->rb_left;
                else if (id > nh->id)
                        pp = &next->rb_right;
                else
                        return nh;
        }
        return NULL;
}
EXPORT_SYMBOL_GPL(nexthop_find_by_id);

/* used for auto id allocation; called with rtnl held */
static u32 nh_find_unused_id(struct net *net)
{
        u32 id_start = net->nexthop.last_id_allocated;

        while (1) {
                net->nexthop.last_id_allocated++;
                if (net->nexthop.last_id_allocated == id_start)
                        break;

                if (!nexthop_find_by_id(net, net->nexthop.last_id_allocated))
                        return net->nexthop.last_id_allocated;
        }
        return 0;
}

static void nh_res_time_set_deadline(unsigned long next_time,
                                     unsigned long *deadline)
{
        if (time_before(next_time, *deadline))
                *deadline = next_time;
}

static clock_t nh_res_table_unbalanced_time(struct nh_res_table *res_table)
{
        if (list_empty(&res_table->uw_nh_entries))
                return 0;
        return jiffies_delta_to_clock_t(jiffies - res_table->unbalanced_since);
}

static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)
{
        struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);
        struct nlattr *nest;

        nest = nla_nest_start(skb, NHA_RES_GROUP);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u16(skb, NHA_RES_GROUP_BUCKETS,
                        res_table->num_nh_buckets) ||
            nla_put_u32(skb, NHA_RES_GROUP_IDLE_TIMER,
                        jiffies_to_clock_t(res_table->idle_timer)) ||
            nla_put_u32(skb, NHA_RES_GROUP_UNBALANCED_TIMER,
                        jiffies_to_clock_t(res_table->unbalanced_timer)) ||
            nla_put_u64_64bit(skb, NHA_RES_GROUP_UNBALANCED_TIME,
                              nh_res_table_unbalanced_time(res_table),
                              NHA_RES_GROUP_PAD))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static void nh_grp_entry_stats_inc(struct nh_grp_entry *nhge)
{
        struct nh_grp_entry_stats *cpu_stats;

        cpu_stats = get_cpu_ptr(nhge->stats);
        u64_stats_update_begin(&cpu_stats->syncp);
        u64_stats_inc(&cpu_stats->packets);
        u64_stats_update_end(&cpu_stats->syncp);
        put_cpu_ptr(cpu_stats);
}

static void nh_grp_entry_stats_read(struct nh_grp_entry *nhge,
                                    u64 *ret_packets)
{
        int i;

        *ret_packets = 0;

        for_each_possible_cpu(i) {
                struct nh_grp_entry_stats *cpu_stats;
                unsigned int start;
                u64 packets;

                cpu_stats = per_cpu_ptr(nhge->stats, i);
                do {
                        start = u64_stats_fetch_begin(&cpu_stats->syncp);
                        packets = u64_stats_read(&cpu_stats->packets);
                } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));

                *ret_packets += packets;
        }
}

static int nh_notifier_grp_hw_stats_init(struct nh_notifier_info *info,
                                         const struct nexthop *nh)
{
        struct nh_group *nhg;
        int i;

        ASSERT_RTNL();
        nhg = rtnl_dereference(nh->nh_grp);

        info->id = nh->id;
        info->type = NH_NOTIFIER_INFO_TYPE_GRP_HW_STATS;
        info->nh_grp_hw_stats = kzalloc(struct_size(info->nh_grp_hw_stats,
                                                    stats, nhg->num_nh),
                                        GFP_KERNEL);
        if (!info->nh_grp_hw_stats)
                return -ENOMEM;

        info->nh_grp_hw_stats->num_nh = nhg->num_nh;
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                info->nh_grp_hw_stats->stats[i].id = nhge->nh->id;
        }

        return 0;
}

static void nh_notifier_grp_hw_stats_fini(struct nh_notifier_info *info)
{
        kfree(info->nh_grp_hw_stats);
}

void nh_grp_hw_stats_report_delta(struct nh_notifier_grp_hw_stats_info *info,
                                  unsigned int nh_idx,
                                  u64 delta_packets)
{
        info->hw_stats_used = true;
        info->stats[nh_idx].packets += delta_packets;
}
EXPORT_SYMBOL(nh_grp_hw_stats_report_delta);

static void nh_grp_hw_stats_apply_update(struct nexthop *nh,
                                         struct nh_notifier_info *info)
{
        struct nh_group *nhg;
        int i;

        ASSERT_RTNL();
        nhg = rtnl_dereference(nh->nh_grp);

        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                nhge->packets_hw += info->nh_grp_hw_stats->stats[i].packets;
        }
}

static int nh_grp_hw_stats_update(struct nexthop *nh, bool *hw_stats_used)
{
        struct nh_notifier_info info = {
                .net = nh->net,
        };
        struct net *net = nh->net;
        int err;

        if (nexthop_notifiers_is_empty(net)) {
                *hw_stats_used = false;
                return 0;
        }

        err = nh_notifier_grp_hw_stats_init(&info, nh);
        if (err)
                return err;

        err = blocking_notifier_call_chain(&net->nexthop.notifier_chain,
                                           NEXTHOP_EVENT_HW_STATS_REPORT_DELTA,
                                           &info);

        /* Cache whatever we got, even if there was an error, otherwise the
         * successful stats retrievals would get lost.
         */
        nh_grp_hw_stats_apply_update(nh, &info);
        *hw_stats_used = info.nh_grp_hw_stats->hw_stats_used;

        nh_notifier_grp_hw_stats_fini(&info);
        return notifier_to_errno(err);
}

static int nla_put_nh_group_stats_entry(struct sk_buff *skb,
                                        struct nh_grp_entry *nhge,
                                        u32 op_flags)
{
        struct nlattr *nest;
        u64 packets;

        nh_grp_entry_stats_read(nhge, &packets);

        nest = nla_nest_start(skb, NHA_GROUP_STATS_ENTRY);
        if (!nest)
                return -EMSGSIZE;

        if (nla_put_u32(skb, NHA_GROUP_STATS_ENTRY_ID, nhge->nh->id) ||
            nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS,
                         packets + nhge->packets_hw))
                goto nla_put_failure;

        if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
            nla_put_uint(skb, NHA_GROUP_STATS_ENTRY_PACKETS_HW,
                         nhge->packets_hw))
                goto nla_put_failure;

        nla_nest_end(skb, nest);
        return 0;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static int nla_put_nh_group_stats(struct sk_buff *skb, struct nexthop *nh,
                                  u32 op_flags)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        struct nlattr *nest;
        bool hw_stats_used;
        int err;
        int i;

        if (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats))
                goto err_out;

        if (op_flags & NHA_OP_FLAG_DUMP_HW_STATS &&
            nhg->hw_stats) {
                err = nh_grp_hw_stats_update(nh, &hw_stats_used);
                if (err)
                        goto out;

                if (nla_put_u32(skb, NHA_HW_STATS_USED, hw_stats_used))
                        goto err_out;
        }

        nest = nla_nest_start(skb, NHA_GROUP_STATS);
        if (!nest)
                goto err_out;

        for (i = 0; i < nhg->num_nh; i++)
                if (nla_put_nh_group_stats_entry(skb, &nhg->nh_entries[i],
                                                 op_flags))
                        goto cancel_out;

        nla_nest_end(skb, nest);
        return 0;

cancel_out:
        nla_nest_cancel(skb, nest);
err_out:
        err = -EMSGSIZE;
out:
        return err;
}

static int nla_put_nh_group(struct sk_buff *skb, struct nexthop *nh,
                            u32 op_flags, u32 *resp_op_flags)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        struct nexthop_grp *p;
        size_t len = nhg->num_nh * sizeof(*p);
        struct nlattr *nla;
        u16 group_type = 0;
        u16 weight;
        int i;

        *resp_op_flags |= NHA_OP_FLAG_RESP_GRP_RESVD_0;

        if (nhg->hash_threshold)
                group_type = NEXTHOP_GRP_TYPE_MPATH;
        else if (nhg->resilient)
                group_type = NEXTHOP_GRP_TYPE_RES;

        if (nla_put_u16(skb, NHA_GROUP_TYPE, group_type))
                goto nla_put_failure;

        nla = nla_reserve(skb, NHA_GROUP, len);
        if (!nla)
                goto nla_put_failure;

        p = nla_data(nla);
        for (i = 0; i < nhg->num_nh; ++i) {
                weight = nhg->nh_entries[i].weight - 1;

                *p++ = (struct nexthop_grp) {
                        .id = nhg->nh_entries[i].nh->id,
                        .weight = weight,
                        .weight_high = weight >> 8,
                };
        }

        if (nhg->resilient && nla_put_nh_group_res(skb, nhg))
                goto nla_put_failure;

        if (op_flags & NHA_OP_FLAG_DUMP_STATS &&
            (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats) ||
             nla_put_nh_group_stats(skb, nh, op_flags)))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -EMSGSIZE;
}

static int nh_fill_node(struct sk_buff *skb, struct nexthop *nh,
                        int event, u32 portid, u32 seq, unsigned int nlflags,
                        u32 op_flags)
{
        struct fib6_nh *fib6_nh;
        struct fib_nh *fib_nh;
        struct nlmsghdr *nlh;
        struct nh_info *nhi;
        struct nhmsg *nhm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
        if (!nlh)
                return -EMSGSIZE;

        nhm = nlmsg_data(nlh);
        nhm->nh_family = AF_UNSPEC;
        nhm->nh_flags = nh->nh_flags;
        nhm->nh_protocol = nh->protocol;
        nhm->nh_scope = 0;
        nhm->resvd = 0;

        if (nla_put_u32(skb, NHA_ID, nh->id))
                goto nla_put_failure;

        if (nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
                u32 resp_op_flags = 0;

                if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))
                        goto nla_put_failure;
                if (nla_put_nh_group(skb, nh, op_flags, &resp_op_flags) ||
                    nla_put_u32(skb, NHA_OP_FLAGS, resp_op_flags))
                        goto nla_put_failure;
                goto out;
        }

        nhi = rtnl_dereference(nh->nh_info);
        nhm->nh_family = nhi->family;
        if (nhi->reject_nh) {
                if (nla_put_flag(skb, NHA_BLACKHOLE))
                        goto nla_put_failure;
                goto out;
        } else if (nhi->fdb_nh) {
                if (nla_put_flag(skb, NHA_FDB))
                        goto nla_put_failure;
        } else {
                const struct net_device *dev;

                dev = nhi->fib_nhc.nhc_dev;
                if (dev && nla_put_u32(skb, NHA_OIF, dev->ifindex))
                        goto nla_put_failure;
        }

        nhm->nh_scope = nhi->fib_nhc.nhc_scope;
        switch (nhi->family) {
        case AF_INET:
                fib_nh = &nhi->fib_nh;
                if (fib_nh->fib_nh_gw_family &&
                    nla_put_be32(skb, NHA_GATEWAY, fib_nh->fib_nh_gw4))
                        goto nla_put_failure;
                break;

        case AF_INET6:
                fib6_nh = &nhi->fib6_nh;
                if (fib6_nh->fib_nh_gw_family &&
                    nla_put_in6_addr(skb, NHA_GATEWAY, &fib6_nh->fib_nh_gw6))
                        goto nla_put_failure;
                break;
        }

        if (nhi->fib_nhc.nhc_lwtstate &&
            lwtunnel_fill_encap(skb, nhi->fib_nhc.nhc_lwtstate,
                                NHA_ENCAP, NHA_ENCAP_TYPE) < 0)
                goto nla_put_failure;

out:
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)
{
        return nla_total_size(0) +        /* NHA_RES_GROUP */
                nla_total_size(2) +        /* NHA_RES_GROUP_BUCKETS */
                nla_total_size(4) +        /* NHA_RES_GROUP_IDLE_TIMER */
                nla_total_size(4) +        /* NHA_RES_GROUP_UNBALANCED_TIMER */
                nla_total_size_64bit(8);/* NHA_RES_GROUP_UNBALANCED_TIME */
}

static size_t nh_nlmsg_size_grp(struct nexthop *nh)
{
        struct nh_group *nhg = rtnl_dereference(nh->nh_grp);
        size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;
        size_t tot = nla_total_size(sz) +
                nla_total_size(2); /* NHA_GROUP_TYPE */

        if (nhg->resilient)
                tot += nh_nlmsg_size_grp_res(nhg);

        return tot;
}

static size_t nh_nlmsg_size_single(struct nexthop *nh)
{
        struct nh_info *nhi = rtnl_dereference(nh->nh_info);
        size_t sz;

        /* covers NHA_BLACKHOLE since NHA_OIF and BLACKHOLE
         * are mutually exclusive
         */
        sz = nla_total_size(4);  /* NHA_OIF */

        switch (nhi->family) {
        case AF_INET:
                if (nhi->fib_nh.fib_nh_gw_family)
                        sz += nla_total_size(4);  /* NHA_GATEWAY */
                break;

        case AF_INET6:
                /* NHA_GATEWAY */
                if (nhi->fib6_nh.fib_nh_gw_family)
                        sz += nla_total_size(sizeof(const struct in6_addr));
                break;
        }

        if (nhi->fib_nhc.nhc_lwtstate) {
                sz += lwtunnel_get_encap_size(nhi->fib_nhc.nhc_lwtstate);
                sz += nla_total_size(2);  /* NHA_ENCAP_TYPE */
        }

        return sz;
}

static size_t nh_nlmsg_size(struct nexthop *nh)
{
        size_t sz = NLMSG_ALIGN(sizeof(struct nhmsg));

        sz += nla_total_size(4); /* NHA_ID */

        if (nh->is_group)
                sz += nh_nlmsg_size_grp(nh) +
                      nla_total_size(4) +        /* NHA_OP_FLAGS */
                      0;
        else
                sz += nh_nlmsg_size_single(nh);

        return sz;
}

static void nexthop_notify(int event, struct nexthop *nh, struct nl_info *info)
{
        unsigned int nlflags = info->nlh ? info->nlh->nlmsg_flags : 0;
        u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = nlmsg_new(nh_nlmsg_size(nh), gfp_any());
        if (!skb)
                goto errout;

        err = nh_fill_node(skb, nh, event, info->portid, seq, nlflags, 0);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in nh_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, info->nl_net, info->portid, RTNLGRP_NEXTHOP,
                    info->nlh, gfp_any());
        return;
errout:
        rtnl_set_sk_err(info->nl_net, RTNLGRP_NEXTHOP, err);
}

static unsigned long nh_res_bucket_used_time(const struct nh_res_bucket *bucket)
{
        return (unsigned long)atomic_long_read(&bucket->used_time);
}

static unsigned long
nh_res_bucket_idle_point(const struct nh_res_table *res_table,
                         const struct nh_res_bucket *bucket,
                         unsigned long now)
{
        unsigned long time = nh_res_bucket_used_time(bucket);

        /* Bucket was not used since it was migrated. The idle time is now. */
        if (time == bucket->migrated_time)
                return now;

        return time + res_table->idle_timer;
}

static unsigned long
nh_res_table_unb_point(const struct nh_res_table *res_table)
{
        return res_table->unbalanced_since + res_table->unbalanced_timer;
}

static void nh_res_bucket_set_idle(const struct nh_res_table *res_table,
                                   struct nh_res_bucket *bucket)
{
        unsigned long now = jiffies;

        atomic_long_set(&bucket->used_time, (long)now);
        bucket->migrated_time = now;
}

static void nh_res_bucket_set_busy(struct nh_res_bucket *bucket)
{
        atomic_long_set(&bucket->used_time, (long)jiffies);
}

static clock_t nh_res_bucket_idle_time(const struct nh_res_bucket *bucket)
{
        unsigned long used_time = nh_res_bucket_used_time(bucket);

        return jiffies_delta_to_clock_t(jiffies - used_time);
}

static int nh_fill_res_bucket(struct sk_buff *skb, struct nexthop *nh,
                              struct nh_res_bucket *bucket, u16 bucket_index,
                              int event, u32 portid, u32 seq,
                              unsigned int nlflags,
                              struct netlink_ext_ack *extack)
{
        struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
        struct nlmsghdr *nlh;
        struct nlattr *nest;
        struct nhmsg *nhm;

        nlh = nlmsg_put(skb, portid, seq, event, sizeof(*nhm), nlflags);
        if (!nlh)
                return -EMSGSIZE;

        nhm = nlmsg_data(nlh);
        nhm->nh_family = AF_UNSPEC;
        nhm->nh_flags = bucket->nh_flags;
        nhm->nh_protocol = nh->protocol;
        nhm->nh_scope = 0;
        nhm->resvd = 0;

        if (nla_put_u32(skb, NHA_ID, nh->id))
                goto nla_put_failure;

        nest = nla_nest_start(skb, NHA_RES_BUCKET);
        if (!nest)
                goto nla_put_failure;

        if (nla_put_u16(skb, NHA_RES_BUCKET_INDEX, bucket_index) ||
            nla_put_u32(skb, NHA_RES_BUCKET_NH_ID, nhge->nh->id) ||
            nla_put_u64_64bit(skb, NHA_RES_BUCKET_IDLE_TIME,
                              nh_res_bucket_idle_time(bucket),
                              NHA_RES_BUCKET_PAD))
                goto nla_put_failure_nest;

        nla_nest_end(skb, nest);
        nlmsg_end(skb, nlh);
        return 0;

nla_put_failure_nest:
        nla_nest_cancel(skb, nest);
nla_put_failure:
        nlmsg_cancel(skb, nlh);
        return -EMSGSIZE;
}

static void nexthop_bucket_notify(struct nh_res_table *res_table,
                                  u16 bucket_index)
{
        struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
        struct nh_grp_entry *nhge = nh_res_dereference(bucket->nh_entry);
        struct nexthop *nh = nhge->nh_parent;
        struct sk_buff *skb;
        int err = -ENOBUFS;

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                goto errout;

        err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
                                 RTM_NEWNEXTHOPBUCKET, 0, 0, NLM_F_REPLACE,
                                 NULL);
        if (err < 0) {
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, nh->net, 0, RTNLGRP_NEXTHOP, NULL, GFP_KERNEL);
        return;
errout:
        rtnl_set_sk_err(nh->net, RTNLGRP_NEXTHOP, err);
}

static bool valid_group_nh(struct nexthop *nh, unsigned int npaths,
                           bool *is_fdb, struct netlink_ext_ack *extack)
{
        if (nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(nh->nh_grp);

                /* Nesting groups within groups is not supported. */
                if (nhg->hash_threshold) {
                        NL_SET_ERR_MSG(extack,
                                       "Hash-threshold group can not be a nexthop within a group");
                        return false;
                }
                if (nhg->resilient) {
                        NL_SET_ERR_MSG(extack,
                                       "Resilient group can not be a nexthop within a group");
                        return false;
                }
                *is_fdb = nhg->fdb_nh;
        } else {
                struct nh_info *nhi = rtnl_dereference(nh->nh_info);

                if (nhi->reject_nh && npaths > 1) {
                        NL_SET_ERR_MSG(extack,
                                       "Blackhole nexthop can not be used in a group with more than 1 path");
                        return false;
                }
                *is_fdb = nhi->fdb_nh;
        }

        return true;
}

static int nh_check_attr_fdb_group(struct nexthop *nh, u8 *nh_family,
                                   struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;

        nhi = rtnl_dereference(nh->nh_info);

        if (!nhi->fdb_nh) {
                NL_SET_ERR_MSG(extack, "FDB nexthop group can only have fdb nexthops");
                return -EINVAL;
        }

        if (*nh_family == AF_UNSPEC) {
                *nh_family = nhi->family;
        } else if (*nh_family != nhi->family) {
                NL_SET_ERR_MSG(extack, "FDB nexthop group cannot have mixed family nexthops");
                return -EINVAL;
        }

        return 0;
}

static int nh_check_attr_group(struct net *net,
                               struct nlattr *tb[], size_t tb_size,
                               u16 nh_grp_type, struct netlink_ext_ack *extack)
{
        unsigned int len = nla_len(tb[NHA_GROUP]);
        struct nexthop_grp *nhg;
        unsigned int i, j;

        if (!len || len & (sizeof(struct nexthop_grp) - 1)) {
                NL_SET_ERR_MSG(extack,
                               "Invalid length for nexthop group attribute");
                return -EINVAL;
        }

        /* convert len to number of nexthop ids */
        len /= sizeof(*nhg);

        nhg = nla_data(tb[NHA_GROUP]);
        for (i = 0; i < len; ++i) {
                if (nhg[i].resvd2) {
                        NL_SET_ERR_MSG(extack, "Reserved field in nexthop_grp must be 0");
                        return -EINVAL;
                }
                if (nexthop_grp_weight(&nhg[i]) == 0) {
                        /* 0xffff got passed in, representing weight of 0x10000,
                         * which is too heavy.
                         */
                        NL_SET_ERR_MSG(extack, "Invalid value for weight");
                        return -EINVAL;
                }
                for (j = i + 1; j < len; ++j) {
                        if (nhg[i].id == nhg[j].id) {
                                NL_SET_ERR_MSG(extack, "Nexthop id can not be used twice in a group");
                                return -EINVAL;
                        }
                }
        }

        nhg = nla_data(tb[NHA_GROUP]);
        for (i = NHA_GROUP_TYPE + 1; i < tb_size; ++i) {
                if (!tb[i])
                        continue;
                switch (i) {
                case NHA_HW_STATS_ENABLE:
                case NHA_FDB:
                        continue;
                case NHA_RES_GROUP:
                        if (nh_grp_type == NEXTHOP_GRP_TYPE_RES)
                                continue;
                        break;
                }
                NL_SET_ERR_MSG(extack,
                               "No other attributes can be set in nexthop groups");
                return -EINVAL;
        }

        return 0;
}

static int nh_check_attr_group_rtnl(struct net *net, struct nlattr *tb[],
                                    struct netlink_ext_ack *extack)
{
        u8 nh_family = AF_UNSPEC;
        struct nexthop_grp *nhg;
        unsigned int len;
        unsigned int i;
        u8 nhg_fdb;

        len = nla_len(tb[NHA_GROUP]) / sizeof(*nhg);
        nhg = nla_data(tb[NHA_GROUP]);
        nhg_fdb = !!tb[NHA_FDB];

        for (i = 0; i < len; i++) {
                struct nexthop *nh;
                bool is_fdb_nh;

                nh = nexthop_find_by_id(net, nhg[i].id);
                if (!nh) {
                        NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                        return -EINVAL;
                }
                if (!valid_group_nh(nh, len, &is_fdb_nh, extack))
                        return -EINVAL;

                if (nhg_fdb && nh_check_attr_fdb_group(nh, &nh_family, extack))
                        return -EINVAL;

                if (!nhg_fdb && is_fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Non FDB nexthop group cannot have fdb nexthops");
                        return -EINVAL;
                }
        }

        return 0;
}

static bool ipv6_good_nh(const struct fib6_nh *nh)
{
        int state = NUD_REACHABLE;
        struct neighbour *n;

        rcu_read_lock();

        n = __ipv6_neigh_lookup_noref_stub(nh->fib_nh_dev, &nh->fib_nh_gw6);
        if (n)
                state = READ_ONCE(n->nud_state);

        rcu_read_unlock();

        return !!(state & NUD_VALID);
}

static bool ipv4_good_nh(const struct fib_nh *nh)
{
        int state = NUD_REACHABLE;
        struct neighbour *n;

        rcu_read_lock();

        n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
                                      (__force u32)nh->fib_nh_gw4);
        if (n)
                state = READ_ONCE(n->nud_state);

        rcu_read_unlock();

        return !!(state & NUD_VALID);
}

static bool nexthop_is_good_nh(const struct nexthop *nh)
{
        struct nh_info *nhi = rcu_dereference(nh->nh_info);

        switch (nhi->family) {
        case AF_INET:
                return ipv4_good_nh(&nhi->fib_nh);
        case AF_INET6:
                return ipv6_good_nh(&nhi->fib6_nh);
        }

        return false;
}

static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)
{
        int i;

        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                if (hash > atomic_read(&nhge->hthr.upper_bound))
                        continue;

                nh_grp_entry_stats_inc(nhge);
                return nhge->nh;
        }

        WARN_ON_ONCE(1);
        return NULL;
}

static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)
{
        struct nh_grp_entry *nhge0 = NULL;
        int i;

        if (nhg->fdb_nh)
                return nexthop_select_path_fdb(nhg, hash);

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                /* nexthops always check if it is good and does
                 * not rely on a sysctl for this behavior
                 */
                if (!nexthop_is_good_nh(nhge->nh))
                        continue;

                if (!nhge0)
                        nhge0 = nhge;

                if (hash > atomic_read(&nhge->hthr.upper_bound))
                        continue;

                nh_grp_entry_stats_inc(nhge);
                return nhge->nh;
        }

        if (!nhge0)
                nhge0 = &nhg->nh_entries[0];
        nh_grp_entry_stats_inc(nhge0);
        return nhge0->nh;
}

static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)
{
        struct nh_res_table *res_table = rcu_dereference(nhg->res_table);
        u16 bucket_index = hash % res_table->num_nh_buckets;
        struct nh_res_bucket *bucket;
        struct nh_grp_entry *nhge;

        /* nexthop_select_path() is expected to return a non-NULL value, so
         * skip protocol validation and just hand out whatever there is.
         */
        bucket = &res_table->nh_buckets[bucket_index];
        nh_res_bucket_set_busy(bucket);
        nhge = rcu_dereference(bucket->nh_entry);
        nh_grp_entry_stats_inc(nhge);
        return nhge->nh;
}

struct nexthop *nexthop_select_path(struct nexthop *nh, int hash)
{
        struct nh_group *nhg;

        if (!nh->is_group)
                return nh;

        nhg = rcu_dereference(nh->nh_grp);
        if (nhg->hash_threshold)
                return nexthop_select_path_hthr(nhg, hash);
        else if (nhg->resilient)
                return nexthop_select_path_res(nhg, hash);

        /* Unreachable. */
        return NULL;
}
EXPORT_SYMBOL_GPL(nexthop_select_path);

int nexthop_for_each_fib6_nh(struct nexthop *nh,
                             int (*cb)(struct fib6_nh *nh, void *arg),
                             void *arg)
{
        struct nh_info *nhi;
        int err;

        if (nh->is_group) {
                struct nh_group *nhg;
                int i;

                nhg = rcu_dereference_rtnl(nh->nh_grp);
                for (i = 0; i < nhg->num_nh; i++) {
                        struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                        nhi = rcu_dereference_rtnl(nhge->nh->nh_info);
                        err = cb(&nhi->fib6_nh, arg);
                        if (err)
                                return err;
                }
        } else {
                nhi = rcu_dereference_rtnl(nh->nh_info);
                err = cb(&nhi->fib6_nh, arg);
                if (err)
                        return err;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(nexthop_for_each_fib6_nh);

static int check_src_addr(const struct in6_addr *saddr,
                          struct netlink_ext_ack *extack)
{
        if (!ipv6_addr_any(saddr)) {
                NL_SET_ERR_MSG(extack, "IPv6 routes using source address can not use nexthop objects");
                return -EINVAL;
        }
        return 0;
}

int fib6_check_nexthop(struct nexthop *nh, struct fib6_config *cfg,
                       struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        bool is_fdb_nh;

        /* fib6_src is unique to a fib6_info and limits the ability to cache
         * routes in fib6_nh within a nexthop that is potentially shared
         * across multiple fib entries. If the config wants to use source
         * routing it can not use nexthop objects. mlxsw also does not allow
         * fib6_src on routes.
         */
        if (cfg && check_src_addr(&cfg->fc_src, extack) < 0)
                return -EINVAL;

        if (nh->is_group) {
                struct nh_group *nhg;

                nhg = rcu_dereference_rtnl(nh->nh_grp);
                if (nhg->has_v4)
                        goto no_v4_nh;
                is_fdb_nh = nhg->fdb_nh;
        } else {
                nhi = rcu_dereference_rtnl(nh->nh_info);
                if (nhi->family == AF_INET)
                        goto no_v4_nh;
                is_fdb_nh = nhi->fdb_nh;
        }

        if (is_fdb_nh) {
                NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                return -EINVAL;
        }

        return 0;
no_v4_nh:
        NL_SET_ERR_MSG(extack, "IPv6 routes can not use an IPv4 nexthop");
        return -EINVAL;
}
EXPORT_SYMBOL_GPL(fib6_check_nexthop);

/* if existing nexthop has ipv6 routes linked to it, need
 * to verify this new spec works with ipv6
 */
static int fib6_check_nh_list(struct nexthop *old, struct nexthop *new,
                              struct netlink_ext_ack *extack)
{
        struct fib6_info *f6i;

        if (list_empty(&old->f6i_list))
                return 0;

        list_for_each_entry(f6i, &old->f6i_list, nh_list) {
                if (check_src_addr(&f6i->fib6_src.addr, extack) < 0)
                        return -EINVAL;
        }

        return fib6_check_nexthop(new, NULL, extack);
}

static int nexthop_check_scope(struct nh_info *nhi, u8 scope,
                               struct netlink_ext_ack *extack)
{
        if (scope == RT_SCOPE_HOST && nhi->fib_nhc.nhc_gw_family) {
                NL_SET_ERR_MSG(extack,
                               "Route with host scope can not have a gateway");
                return -EINVAL;
        }

        if (nhi->fib_nhc.nhc_flags & RTNH_F_ONLINK && scope >= RT_SCOPE_LINK) {
                NL_SET_ERR_MSG(extack, "Scope mismatch with nexthop");
                return -EINVAL;
        }

        return 0;
}

/* Invoked by fib add code to verify nexthop by id is ok with
 * config for prefix; parts of fib_check_nh not done when nexthop
 * object is used.
 */
int fib_check_nexthop(struct nexthop *nh, u8 scope,
                      struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        int err = 0;

        if (nh->is_group) {
                struct nh_group *nhg;

                nhg = rtnl_dereference(nh->nh_grp);
                if (nhg->fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                        err = -EINVAL;
                        goto out;
                }

                if (scope == RT_SCOPE_HOST) {
                        NL_SET_ERR_MSG(extack, "Route with host scope can not have multiple nexthops");
                        err = -EINVAL;
                        goto out;
                }

                /* all nexthops in a group have the same scope */
                nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);
                err = nexthop_check_scope(nhi, scope, extack);
        } else {
                nhi = rtnl_dereference(nh->nh_info);
                if (nhi->fdb_nh) {
                        NL_SET_ERR_MSG(extack, "Route cannot point to a fdb nexthop");
                        err = -EINVAL;
                        goto out;
                }
                err = nexthop_check_scope(nhi, scope, extack);
        }

out:
        return err;
}

static int fib_check_nh_list(struct nexthop *old, struct nexthop *new,
                             struct netlink_ext_ack *extack)
{
        struct fib_info *fi;

        list_for_each_entry(fi, &old->fi_list, nh_list) {
                int err;

                err = fib_check_nexthop(new, fi->fib_scope, extack);
                if (err)
                        return err;
        }
        return 0;
}

static bool nh_res_nhge_is_balanced(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets == nhge->res.wants_buckets;
}

static bool nh_res_nhge_is_ow(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets > nhge->res.wants_buckets;
}

static bool nh_res_nhge_is_uw(const struct nh_grp_entry *nhge)
{
        return nhge->res.count_buckets < nhge->res.wants_buckets;
}

static bool nh_res_table_is_balanced(const struct nh_res_table *res_table)
{
        return list_empty(&res_table->uw_nh_entries);
}

static void nh_res_bucket_unset_nh(struct nh_res_bucket *bucket)
{
        struct nh_grp_entry *nhge;

        if (bucket->occupied) {
                nhge = nh_res_dereference(bucket->nh_entry);
                nhge->res.count_buckets--;
                bucket->occupied = false;
        }
}

static void nh_res_bucket_set_nh(struct nh_res_bucket *bucket,
                                 struct nh_grp_entry *nhge)
{
        nh_res_bucket_unset_nh(bucket);

        bucket->occupied = true;
        rcu_assign_pointer(bucket->nh_entry, nhge);
        nhge->res.count_buckets++;
}

static bool nh_res_bucket_should_migrate(struct nh_res_table *res_table,
                                         struct nh_res_bucket *bucket,
                                         unsigned long *deadline, bool *force)
{
        unsigned long now = jiffies;
        struct nh_grp_entry *nhge;
        unsigned long idle_point;

        if (!bucket->occupied) {
                /* The bucket is not occupied, its NHGE pointer is either
                 * NULL or obsolete. We _have to_ migrate: set force.
                 */
                *force = true;
                return true;
        }

        nhge = nh_res_dereference(bucket->nh_entry);

        /* If the bucket is populated by an underweight or balanced
         * nexthop, do not migrate.
         */
        if (!nh_res_nhge_is_ow(nhge))
                return false;

        /* At this point we know that the bucket is populated with an
         * overweight nexthop. It needs to be migrated to a new nexthop if
         * the idle timer of unbalanced timer expired.
         */

        idle_point = nh_res_bucket_idle_point(res_table, bucket, now);
        if (time_after_eq(now, idle_point)) {
                /* The bucket is idle. We _can_ migrate: unset force. */
                *force = false;
                return true;
        }

        /* Unbalanced timer of 0 means "never force". */
        if (res_table->unbalanced_timer) {
                unsigned long unb_point;

                unb_point = nh_res_table_unb_point(res_table);
                if (time_after(now, unb_point)) {
                        /* The bucket is not idle, but the unbalanced timer
                         * expired. We _can_ migrate, but set force anyway,
                         * so that drivers know to ignore activity reports
                         * from the HW.
                         */
                        *force = true;
                        return true;
                }

                nh_res_time_set_deadline(unb_point, deadline);
        }

        nh_res_time_set_deadline(idle_point, deadline);
        return false;
}

static bool nh_res_bucket_migrate(struct nh_res_table *res_table,
                                  u16 bucket_index, bool notify,
                                  bool notify_nl, bool force)
{
        struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index];
        struct nh_grp_entry *new_nhge;
        struct netlink_ext_ack extack;
        int err;

        new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries,
                                            struct nh_grp_entry,
                                            res.uw_nh_entry);
        if (WARN_ON_ONCE(!new_nhge))
                /* If this function is called, "bucket" is either not
                 * occupied, or it belongs to a next hop that is
                 * overweight. In either case, there ought to be a
                 * corresponding underweight next hop.
                 */
                return false;

        if (notify) {
                struct nh_grp_entry *old_nhge;

                old_nhge = nh_res_dereference(bucket->nh_entry);
                err = call_nexthop_res_bucket_notifiers(res_table->net,
                                                        res_table->nhg_id,
                                                        bucket_index, force,
                                                        old_nhge->nh,
                                                        new_nhge->nh, &extack);
                if (err) {
                        pr_err_ratelimited("%s\n", extack._msg);
                        if (!force)
                                return false;
                        /* It is not possible to veto a forced replacement, so
                         * just clear the hardware flags from the nexthop
                         * bucket to indicate to user space that this bucket is
                         * not correctly populated in hardware.
                         */
                        bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
                }
        }

        nh_res_bucket_set_nh(bucket, new_nhge);
        nh_res_bucket_set_idle(res_table, bucket);

        if (notify_nl)
                nexthop_bucket_notify(res_table, bucket_index);

        if (nh_res_nhge_is_balanced(new_nhge))
                list_del(&new_nhge->res.uw_nh_entry);
        return true;
}

#define NH_RES_UPKEEP_DW_MINIMUM_INTERVAL (HZ / 2)

static void nh_res_table_upkeep(struct nh_res_table *res_table,
                                bool notify, bool notify_nl)
{
        unsigned long now = jiffies;
        unsigned long deadline;
        u16 i;

        /* Deadline is the next time that upkeep should be run. It is the
         * earliest time at which one of the buckets might be migrated.
         * Start at the most pessimistic estimate: either unbalanced_timer
         * from now, or if there is none, idle_timer from now. For each
         * encountered time point, call nh_res_time_set_deadline() to
         * refine the estimate.
         */
        if (res_table->unbalanced_timer)
                deadline = now + res_table->unbalanced_timer;
        else
                deadline = now + res_table->idle_timer;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                bool force;

                if (nh_res_bucket_should_migrate(res_table, bucket,
                                                 &deadline, &force)) {
                        if (!nh_res_bucket_migrate(res_table, i, notify,
                                                   notify_nl, force)) {
                                unsigned long idle_point;

                                /* A driver can override the migration
                                 * decision if the HW reports that the
                                 * bucket is actually not idle. Therefore
                                 * remark the bucket as busy again and
                                 * update the deadline.
                                 */
                                nh_res_bucket_set_busy(bucket);
                                idle_point = nh_res_bucket_idle_point(res_table,
                                                                      bucket,
                                                                      now);
                                nh_res_time_set_deadline(idle_point, &deadline);
                        }
                }
        }

        /* If the group is still unbalanced, schedule the next upkeep to
         * either the deadline computed above, or the minimum deadline,
         * whichever comes later.
         */
        if (!nh_res_table_is_balanced(res_table)) {
                unsigned long now = jiffies;
                unsigned long min_deadline;

                min_deadline = now + NH_RES_UPKEEP_DW_MINIMUM_INTERVAL;
                if (time_before(deadline, min_deadline))
                        deadline = min_deadline;

                queue_delayed_work(system_power_efficient_wq,
                                   &res_table->upkeep_dw, deadline - now);
        }
}

static void nh_res_table_upkeep_dw(struct work_struct *work)
{
        struct delayed_work *dw = to_delayed_work(work);
        struct nh_res_table *res_table;

        res_table = container_of(dw, struct nh_res_table, upkeep_dw);
        nh_res_table_upkeep(res_table, true, true);
}

static void nh_res_table_cancel_upkeep(struct nh_res_table *res_table)
{
        cancel_delayed_work_sync(&res_table->upkeep_dw);
}

static void nh_res_group_rebalance(struct nh_group *nhg,
                                   struct nh_res_table *res_table)
{
        u16 prev_upper_bound = 0;
        u32 total = 0;
        u32 w = 0;
        int i;

        INIT_LIST_HEAD(&res_table->uw_nh_entries);

        for (i = 0; i < nhg->num_nh; ++i)
                total += nhg->nh_entries[i].weight;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                u16 upper_bound;
                u64 btw;

                w += nhge->weight;
                btw = ((u64)res_table->num_nh_buckets) * w;
                upper_bound = DIV_ROUND_CLOSEST_ULL(btw, total);
                nhge->res.wants_buckets = upper_bound - prev_upper_bound;
                prev_upper_bound = upper_bound;

                if (nh_res_nhge_is_uw(nhge)) {
                        if (list_empty(&res_table->uw_nh_entries))
                                res_table->unbalanced_since = jiffies;
                        list_add(&nhge->res.uw_nh_entry,
                                 &res_table->uw_nh_entries);
                }
        }
}

/* Migrate buckets in res_table so that they reference NHGE's from NHG with
 * the right NH ID. Set those buckets that do not have a corresponding NHGE
 * entry in NHG as not occupied.
 */
static void nh_res_table_migrate_buckets(struct nh_res_table *res_table,
                                         struct nh_group *nhg)
{
        u16 i;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                u32 id = rtnl_dereference(bucket->nh_entry)->nh->id;
                bool found = false;
                int j;

                for (j = 0; j < nhg->num_nh; j++) {
                        struct nh_grp_entry *nhge = &nhg->nh_entries[j];

                        if (nhge->nh->id == id) {
                                nh_res_bucket_set_nh(bucket, nhge);
                                found = true;
                                break;
                        }
                }

                if (!found)
                        nh_res_bucket_unset_nh(bucket);
        }
}

static void replace_nexthop_grp_res(struct nh_group *oldg,
                                    struct nh_group *newg)
{
        /* For NH group replacement, the new NHG might only have a stub
         * hash table with 0 buckets, because the number of buckets was not
         * specified. For NH removal, oldg and newg both reference the same
         * res_table. So in any case, in the following, we want to work
         * with oldg->res_table.
         */
        struct nh_res_table *old_res_table = rtnl_dereference(oldg->res_table);
        unsigned long prev_unbalanced_since = old_res_table->unbalanced_since;
        bool prev_has_uw = !list_empty(&old_res_table->uw_nh_entries);

        nh_res_table_cancel_upkeep(old_res_table);
        nh_res_table_migrate_buckets(old_res_table, newg);
        nh_res_group_rebalance(newg, old_res_table);
        if (prev_has_uw && !list_empty(&old_res_table->uw_nh_entries))
                old_res_table->unbalanced_since = prev_unbalanced_since;
        nh_res_table_upkeep(old_res_table, true, false);
}

static void nh_hthr_group_rebalance(struct nh_group *nhg)
{
        u32 total = 0;
        u32 w = 0;
        int i;

        for (i = 0; i < nhg->num_nh; ++i)
                total += nhg->nh_entries[i].weight;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                u32 upper_bound;

                w += nhge->weight;
                upper_bound = DIV_ROUND_CLOSEST_ULL((u64)w << 31, total) - 1;
                atomic_set(&nhge->hthr.upper_bound, upper_bound);
        }
}

static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
                                struct nl_info *nlinfo)
{
        struct nh_grp_entry *nhges, *new_nhges;
        struct nexthop *nhp = nhge->nh_parent;
        struct netlink_ext_ack extack;
        struct nexthop *nh = nhge->nh;
        struct nh_group *nhg, *newg;
        int i, j, err;

        WARN_ON(!nh);

        nhg = rtnl_dereference(nhp->nh_grp);
        newg = nhg->spare;

        /* last entry, keep it visible and remove the parent */
        if (nhg->num_nh == 1) {
                remove_nexthop(net, nhp, nlinfo);
                return;
        }

        newg->has_v4 = false;
        newg->is_multipath = nhg->is_multipath;
        newg->hash_threshold = nhg->hash_threshold;
        newg->resilient = nhg->resilient;
        newg->fdb_nh = nhg->fdb_nh;
        newg->num_nh = nhg->num_nh;

        /* copy old entries to new except the one getting removed */
        nhges = nhg->nh_entries;
        new_nhges = newg->nh_entries;
        for (i = 0, j = 0; i < nhg->num_nh; ++i) {
                struct nh_info *nhi;

                /* current nexthop getting removed */
                if (nhg->nh_entries[i].nh == nh) {
                        newg->num_nh--;
                        continue;
                }

                nhi = rtnl_dereference(nhges[i].nh->nh_info);
                if (nhi->family == AF_INET)
                        newg->has_v4 = true;

                list_del(&nhges[i].nh_list);
                new_nhges[j].stats = nhges[i].stats;
                new_nhges[j].nh_parent = nhges[i].nh_parent;
                new_nhges[j].nh = nhges[i].nh;
                new_nhges[j].weight = nhges[i].weight;
                list_add(&new_nhges[j].nh_list, &new_nhges[j].nh->grp_list);
                j++;
        }

        if (newg->hash_threshold)
                nh_hthr_group_rebalance(newg);
        else if (newg->resilient)
                replace_nexthop_grp_res(nhg, newg);

        rcu_assign_pointer(nhp->nh_grp, newg);

        list_del(&nhge->nh_list);
        free_percpu(nhge->stats);
        nexthop_put(nhge->nh);

        /* Removal of a NH from a resilient group is notified through
         * bucket notifications.
         */
        if (newg->hash_threshold) {
                err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, nhp,
                                             &extack);
                if (err)
                        pr_err("%s\n", extack._msg);
        }

        if (nlinfo)
                nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
}

static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
                                       struct nl_info *nlinfo)
{
        struct nh_grp_entry *nhge, *tmp;

        list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
                remove_nh_grp_entry(net, nhge, nlinfo);

        /* make sure all see the newly published array before releasing rtnl */
        synchronize_net();
}

static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
{
        struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);
        struct nh_res_table *res_table;
        int i, num_nh = nhg->num_nh;

        for (i = 0; i < num_nh; ++i) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];

                if (WARN_ON(!nhge->nh))
                        continue;

                list_del_init(&nhge->nh_list);
        }

        if (nhg->resilient) {
                res_table = rtnl_dereference(nhg->res_table);
                nh_res_table_cancel_upkeep(res_table);
        }
}

/* not called for nexthop replace */
static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
{
        struct fib6_info *f6i;
        bool do_flush = false;
        struct fib_info *fi;

        list_for_each_entry(fi, &nh->fi_list, nh_list) {
                fi->fib_flags |= RTNH_F_DEAD;
                do_flush = true;
        }
        if (do_flush)
                fib_flush(net);

        spin_lock_bh(&nh->lock);

        nh->dead = true;

        while (!list_empty(&nh->f6i_list)) {
                f6i = list_first_entry(&nh->f6i_list, typeof(*f6i), nh_list);

                /* __ip6_del_rt does a release, so do a hold here */
                fib6_info_hold(f6i);

                spin_unlock_bh(&nh->lock);
                ipv6_stub->ip6_del_rt(net, f6i,
                                      !READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode));

                spin_lock_bh(&nh->lock);
        }

        spin_unlock_bh(&nh->lock);
}

static void __remove_nexthop(struct net *net, struct nexthop *nh,
                             struct nl_info *nlinfo)
{
        __remove_nexthop_fib(net, nh);

        if (nh->is_group) {
                remove_nexthop_group(nh, nlinfo);
        } else {
                struct nh_info *nhi;

                nhi = rtnl_dereference(nh->nh_info);
                if (nhi->fib_nhc.nhc_dev)
                        hlist_del(&nhi->dev_hash);

                remove_nexthop_from_groups(net, nh, nlinfo);
        }
}

static void remove_nexthop(struct net *net, struct nexthop *nh,
                           struct nl_info *nlinfo)
{
        call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);

        /* remove from the tree */
        rb_erase(&nh->rb_node, &net->nexthop.rb_root);

        if (nlinfo)
                nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);

        __remove_nexthop(net, nh, nlinfo);
        nh_base_seq_inc(net);

        nexthop_put(nh);
}

/* if any FIB entries reference this nexthop, any dst entries
 * need to be regenerated
 */
static void nh_rt_cache_flush(struct net *net, struct nexthop *nh,
                              struct nexthop *replaced_nh)
{
        struct fib6_info *f6i;
        struct nh_group *nhg;
        int i;

        if (!list_empty(&nh->fi_list))
                rt_cache_flush(net);

        list_for_each_entry(f6i, &nh->f6i_list, nh_list)
                ipv6_stub->fib6_update_sernum(net, f6i);

        /* if an IPv6 group was replaced, we have to release all old
         * dsts to make sure all refcounts are released
         */
        if (!replaced_nh->is_group)
                return;

        nhg = rtnl_dereference(replaced_nh->nh_grp);
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_grp_entry *nhge = &nhg->nh_entries[i];
                struct nh_info *nhi = rtnl_dereference(nhge->nh->nh_info);

                if (nhi->family == AF_INET6)
                        ipv6_stub->fib6_nh_release_dsts(&nhi->fib6_nh);
        }
}

static int replace_nexthop_grp(struct net *net, struct nexthop *old,
                               struct nexthop *new, const struct nh_config *cfg,
                               struct netlink_ext_ack *extack)
{
        struct nh_res_table *tmp_table = NULL;
        struct nh_res_table *new_res_table;
        struct nh_res_table *old_res_table;
        struct nh_group *oldg, *newg;
        int i, err;

        if (!new->is_group) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with a nexthop.");
                return -EINVAL;
        }

        oldg = rtnl_dereference(old->nh_grp);
        newg = rtnl_dereference(new->nh_grp);

        if (newg->hash_threshold != oldg->hash_threshold) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop group with one of a different type.");
                return -EINVAL;
        }

        if (newg->hash_threshold) {
                err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new,
                                             extack);
                if (err)
                        return err;
        } else if (newg->resilient) {
                new_res_table = rtnl_dereference(newg->res_table);
                old_res_table = rtnl_dereference(oldg->res_table);

                /* Accept if num_nh_buckets was not given, but if it was
                 * given, demand that the value be correct.
                 */
                if (cfg->nh_grp_res_has_num_buckets &&
                    cfg->nh_grp_res_num_buckets !=
                    old_res_table->num_nh_buckets) {
                        NL_SET_ERR_MSG(extack, "Can not change number of buckets of a resilient nexthop group.");
                        return -EINVAL;
                }

                /* Emit a pre-replace notification so that listeners could veto
                 * a potentially unsupported configuration. Otherwise,
                 * individual bucket replacement notifications would need to be
                 * vetoed, which is something that should only happen if the
                 * bucket is currently active.
                 */
                err = call_nexthop_res_table_notifiers(net, new, extack);
                if (err)
                        return err;

                if (cfg->nh_grp_res_has_idle_timer)
                        old_res_table->idle_timer = cfg->nh_grp_res_idle_timer;
                if (cfg->nh_grp_res_has_unbalanced_timer)
                        old_res_table->unbalanced_timer =
                                cfg->nh_grp_res_unbalanced_timer;

                replace_nexthop_grp_res(oldg, newg);

                tmp_table = new_res_table;
                rcu_assign_pointer(newg->res_table, old_res_table);
                rcu_assign_pointer(newg->spare->res_table, old_res_table);
        }

        /* update parents - used by nexthop code for cleanup */
        for (i = 0; i < newg->num_nh; i++)
                newg->nh_entries[i].nh_parent = old;

        rcu_assign_pointer(old->nh_grp, newg);

        /* Make sure concurrent readers are not using 'oldg' anymore. */
        synchronize_net();

        if (newg->resilient) {
                rcu_assign_pointer(oldg->res_table, tmp_table);
                rcu_assign_pointer(oldg->spare->res_table, tmp_table);
        }

        for (i = 0; i < oldg->num_nh; i++)
                oldg->nh_entries[i].nh_parent = new;

        rcu_assign_pointer(new->nh_grp, oldg);

        return 0;
}

static void nh_group_v4_update(struct nh_group *nhg)
{
        struct nh_grp_entry *nhges;
        bool has_v4 = false;
        int i;

        nhges = nhg->nh_entries;
        for (i = 0; i < nhg->num_nh; i++) {
                struct nh_info *nhi;

                nhi = rtnl_dereference(nhges[i].nh->nh_info);
                if (nhi->family == AF_INET)
                        has_v4 = true;
        }
        nhg->has_v4 = has_v4;
}

static int replace_nexthop_single_notify_res(struct net *net,
                                             struct nh_res_table *res_table,
                                             struct nexthop *old,
                                             struct nh_info *oldi,
                                             struct nh_info *newi,
                                             struct netlink_ext_ack *extack)
{
        u32 nhg_id = res_table->nhg_id;
        int err;
        u16 i;

        for (i = 0; i < res_table->num_nh_buckets; i++) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;

                nhge = rtnl_dereference(bucket->nh_entry);
                if (nhge->nh == old) {
                        err = __call_nexthop_res_bucket_notifiers(net, nhg_id,
                                                                  i, true,
                                                                  oldi, newi,
                                                                  extack);
                        if (err)
                                goto err_notify;
                }
        }

        return 0;

err_notify:
        while (i-- > 0) {
                struct nh_res_bucket *bucket = &res_table->nh_buckets[i];
                struct nh_grp_entry *nhge;

                nhge = rtnl_dereference(bucket->nh_entry);
                if (nhge->nh == old)
                        __call_nexthop_res_bucket_notifiers(net, nhg_id, i,
                                                            true, newi, oldi,
                                                            extack);
        }
        return err;
}

static int replace_nexthop_single_notify(struct net *net,
                                         struct nexthop *group_nh,
                                         struct nexthop *old,
                                         struct nh_info *oldi,
                                         struct nh_info *newi,
                                         struct netlink_ext_ack *extack)
{
        struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);
        struct nh_res_table *res_table;

        if (nhg->hash_threshold) {
                return call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE,
                                              group_nh, extack);
        } else if (nhg->resilient) {
                res_table = rtnl_dereference(nhg->res_table);
                return replace_nexthop_single_notify_res(net, res_table,
                                                         old, oldi, newi,
                                                         extack);
        }

        return -EINVAL;
}

static int replace_nexthop_single(struct net *net, struct nexthop *old,
                                  struct nexthop *new,
                                  struct netlink_ext_ack *extack)
{
        u8 old_protocol, old_nh_flags;
        struct nh_info *oldi, *newi;
        struct nh_grp_entry *nhge;
        int err;

        if (new->is_group) {
                NL_SET_ERR_MSG(extack, "Can not replace a nexthop with a nexthop group.");
                return -EINVAL;
        }

        err = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new, extack);
        if (err)
                return err;

        /* Hardware flags were set on 'old' as 'new' is not in the red-black
         * tree. Therefore, inherit the flags from 'old' to 'new'.
         */
        new->nh_flags |= old->nh_flags & (RTNH_F_OFFLOAD | RTNH_F_TRAP);

        oldi = rtnl_dereference(old->nh_info);
        newi = rtnl_dereference(new->nh_info);

        newi->nh_parent = old;
        oldi->nh_parent = new;

        old_protocol = old->protocol;
        old_nh_flags = old->nh_flags;

        old->protocol = new->protocol;
        old->nh_flags = new->nh_flags;

        rcu_assign_pointer(old->nh_info, newi);
        rcu_assign_pointer(new->nh_info, oldi);

        /* Send a replace notification for all the groups using the nexthop. */
        list_for_each_entry(nhge, &old->grp_list, nh_list) {
                struct nexthop *nhp = nhge->nh_parent;

                err = replace_nexthop_single_notify(net, nhp, old, oldi, newi,
                                                    extack);
                if (err)
                        goto err_notify;
        }

        /* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially
         * update IPv4 indication in all the groups using the nexthop.
         */
        if (oldi->family == AF_INET && newi->family == AF_INET6) {
                list_for_each_entry(nhge, &old->grp_list, nh_list) {
                        struct nexthop *nhp = nhge->nh_parent;
                        struct nh_group *nhg;

                        nhg = rtnl_dereference(nhp->nh_grp);
                        nh_group_v4_update(nhg);
                }
        }

        return 0;

err_notify:
        rcu_assign_pointer(new->nh_info, newi);
        rcu_assign_pointer(old->nh_info, oldi);
        old->nh_flags = old_nh_flags;
        old->protocol = old_protocol;
        oldi->nh_parent = old;
        newi->nh_parent = new;
        list_for_each_entry_continue_reverse(nhge, &old->grp_list, nh_list) {
                struct nexthop *nhp = nhge->nh_parent;

                replace_nexthop_single_notify(net, nhp, old, newi, oldi, NULL);
        }
        call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, old, extack);
        return err;
}

static void __nexthop_replace_notify(struct net *net, struct nexthop *nh,
                                     struct nl_info *info)
{
        struct fib6_info *f6i;

        if (!list_empty(&nh->fi_list)) {
                struct fib_info *fi;

                /* expectation is a few fib_info per nexthop and then
                 * a lot of routes per fib_info. So mark the fib_info
                 * and then walk the fib tables once
                 */
                list_for_each_entry(fi, &nh->fi_list, nh_list)
                        fi->nh_updated = true;

                fib_info_notify_update(net, info);

                list_for_each_entry(fi, &nh->fi_list, nh_list)
                        fi->nh_updated = false;
        }

        list_for_each_entry(f6i, &nh->f6i_list, nh_list)
                ipv6_stub->fib6_rt_update(net, f6i, info);
}

/* send RTM_NEWROUTE with REPLACE flag set for all FIB entries
 * linked to this nexthop and for all groups that the nexthop
 * is a member of
 */
static void nexthop_replace_notify(struct net *net, struct nexthop *nh,
                                   struct nl_info *info)
{
        struct nh_grp_entry *nhge;

        __nexthop_replace_notify(net, nh, info);

        list_for_each_entry(nhge, &nh->grp_list, nh_list)
                __nexthop_replace_notify(net, nhge->nh_parent, info);
}

static int replace_nexthop(struct net *net, struct nexthop *old,
                           struct nexthop *new, const struct nh_config *cfg,
                           struct netlink_ext_ack *extack)
{
        bool new_is_reject = false;
        struct nh_grp_entry *nhge;
        int err;

        /* check that existing FIB entries are ok with the
         * new nexthop definition
         */
        err = fib_check_nh_list(old, new, extack);
        if (err)
                return err;

        err = fib6_check_nh_list(old, new, extack);
        if (err)
                return err;

        if (!new->is_group) {
                struct nh_info *nhi = rtnl_dereference(new->nh_info);

                new_is_reject = nhi->reject_nh;
        }

        list_for_each_entry(nhge, &old->grp_list, nh_list) {
                /* if new nexthop is a blackhole, any groups using this
                 * nexthop cannot have more than 1 path
                 */
                if (new_is_reject &&
                    nexthop_num_path(nhge->nh_parent) > 1) {
                        NL_SET_ERR_MSG(extack, "Blackhole nexthop can not be a member of a group with more than one path");
                        return -EINVAL;
                }

                err = fib_check_nh_list(nhge->nh_parent, new, extack);
                if (err)
                        return err;

                err = fib6_check_nh_list(nhge->nh_parent, new, extack);
                if (err)
                        return err;
        }

        if (old->is_group)
                err = replace_nexthop_grp(net, old, new, cfg, extack);
        else
                err = replace_nexthop_single(net, old, new, extack);

        if (!err) {
                nh_rt_cache_flush(net, old, new);

                __remove_nexthop(net, new, NULL);
                nexthop_put(new);
        }

        return err;
}

/* called with rtnl_lock held */
static int insert_nexthop(struct net *net, struct nexthop *new_nh,
                          struct nh_config *cfg, struct netlink_ext_ack *extack)
{
        struct rb_node **pp, *parent = NULL, *next;
        struct rb_root *root = &net->nexthop.rb_root;
        bool replace = !!(cfg->nlflags & NLM_F_REPLACE);
        bool create = !!(cfg->nlflags & NLM_F_CREATE);
        u32 new_id = new_nh->id;
        int replace_notify = 0;
        int rc = -EEXIST;

        pp = &root->rb_node;
        while (1) {
                struct nexthop *nh;

                next = *pp;
                if (!next)
                        break;

                parent = next;

                nh = rb_entry(parent, struct nexthop, rb_node);
                if (new_id < nh->id) {
                        pp = &next->rb_left;
                } else if (new_id > nh->id) {
                        pp = &next->rb_right;
                } else if (replace) {
                        rc = replace_nexthop(net, nh, new_nh, cfg, extack);
                        if (!rc) {
                                new_nh = nh; /* send notification with old nh */
                                replace_notify = 1;
                        }
                        goto out;
                } else {
                        /* id already exists and not a replace */
                        goto out;
                }
        }

        if (replace && !create) {
                NL_SET_ERR_MSG(extack, "Replace specified without create and no entry exists");
                rc = -ENOENT;
                goto out;
        }

        if (new_nh->is_group) {
                struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);
                struct nh_res_table *res_table;

                if (nhg->resilient) {
                        res_table = rtnl_dereference(nhg->res_table);

                        /* Not passing the number of buckets is OK when
                         * replacing, but not when creating a new group.
                         */
                        if (!cfg->nh_grp_res_has_num_buckets) {
                                NL_SET_ERR_MSG(extack, "Number of buckets not specified for nexthop group insertion");
                                rc = -EINVAL;
                                goto out;
                        }

                        nh_res_group_rebalance(nhg, res_table);

                        /* Do not send bucket notifications, we do full
                         * notification below.
                         */
                        nh_res_table_upkeep(res_table, false, false);
                }
        }

        rb_link_node_rcu(&new_nh->rb_node, parent, pp);
        rb_insert_color(&new_nh->rb_node, root);

        /* The initial insertion is a full notification for hash-threshold as
         * well as resilient groups.
         */
        rc = call_nexthop_notifiers(net, NEXTHOP_EVENT_REPLACE, new_nh, extack);
        if (rc)
                rb_erase(&new_nh->rb_node, &net->nexthop.rb_root);

out:
        if (!rc) {
                nh_base_seq_inc(net);
                nexthop_notify(RTM_NEWNEXTHOP, new_nh, &cfg->nlinfo);
                if (replace_notify &&
                    READ_ONCE(net->ipv4.sysctl_nexthop_compat_mode))
                        nexthop_replace_notify(net, new_nh, &cfg->nlinfo);
        }

        return rc;
}

/* rtnl */
/* remove all nexthops tied to a device being deleted */
static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
{
        unsigned int hash = nh_dev_hashfn(dev->ifindex);
        struct net *net = dev_net(dev);
        struct hlist_head *head = &net->nexthop.devhash[hash];
        struct hlist_node *n;
        struct nh_info *nhi;

        hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
                if (nhi->fib_nhc.nhc_dev != dev)
                        continue;

                if (nhi->reject_nh &&
                    (event == NETDEV_DOWN || event == NETDEV_CHANGE))
                        continue;

                remove_nexthop(net, nhi->nh_parent, NULL);
        }
}

/* rtnl; called when net namespace is deleted */
static void flush_all_nexthops(struct net *net)
{
        struct rb_root *root = &net->nexthop.rb_root;
        struct rb_node *node;
        struct nexthop *nh;

        while ((node = rb_first(root))) {
                nh = rb_entry(node, struct nexthop, rb_node);
                remove_nexthop(net, nh, NULL);
                cond_resched();
        }
}

static struct nexthop *nexthop_create_group(struct net *net,
                                            struct nh_config *cfg)
{
        struct nlattr *grps_attr = cfg->nh_grp;
        struct nexthop_grp *entry = nla_data(grps_attr);
        u16 num_nh = nla_len(grps_attr) / sizeof(*entry);
        struct nh_group *nhg;
        struct nexthop *nh;
        int err;
        int i;

        nh = nexthop_alloc();
        if (!nh)
                return ERR_PTR(-ENOMEM);

        nh->is_group = 1;

        nhg = nexthop_grp_alloc(num_nh);
        if (!nhg) {
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }

        /* spare group used for removals */
        nhg->spare = nexthop_grp_alloc(num_nh);
        if (!nhg->spare) {
                kfree(nhg);
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }
        nhg->spare->spare = nhg;

        for (i = 0; i < nhg->num_nh; ++i) {
                struct nexthop *nhe;
                struct nh_info *nhi;

                nhe = nexthop_find_by_id(net, entry[i].id);
                if (!nexthop_get(nhe)) {
                        err = -ENOENT;
                        goto out_no_nh;
                }

                nhi = rtnl_dereference(nhe->nh_info);
                if (nhi->family == AF_INET)
                        nhg->has_v4 = true;

                nhg->nh_entries[i].stats =
                        netdev_alloc_pcpu_stats(struct nh_grp_entry_stats);
                if (!nhg->nh_entries[i].stats) {
                        err = -ENOMEM;
                        nexthop_put(nhe);
                        goto out_no_nh;
                }
                nhg->nh_entries[i].nh = nhe;
                nhg->nh_entries[i].weight = nexthop_grp_weight(&entry[i]);

                list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);
                nhg->nh_entries[i].nh_parent = nh;
        }

        if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_MPATH) {
                nhg->hash_threshold = 1;
                nhg->is_multipath = true;
        } else if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES) {
                struct nh_res_table *res_table;

                res_table = nexthop_res_table_alloc(net, cfg->nh_id, cfg);
                if (!res_table) {
                        err = -ENOMEM;
                        goto out_no_nh;
                }

                rcu_assign_pointer(nhg->spare->res_table, res_table);
                rcu_assign_pointer(nhg->res_table, res_table);
                nhg->resilient = true;
                nhg->is_multipath = true;
        }

        WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);

        if (nhg->hash_threshold)
                nh_hthr_group_rebalance(nhg);

        if (cfg->nh_fdb)
                nhg->fdb_nh = 1;

        if (cfg->nh_hw_stats)
                nhg->hw_stats = true;

        rcu_assign_pointer(nh->nh_grp, nhg);

        return nh;

out_no_nh:
        for (i--; i >= 0; --i) {
                list_del(&nhg->nh_entries[i].nh_list);
                free_percpu(nhg->nh_entries[i].stats);
                nexthop_put(nhg->nh_entries[i].nh);
        }

        kfree(nhg->spare);
        kfree(nhg);
        kfree(nh);

        return ERR_PTR(err);
}

static int nh_create_ipv4(struct net *net, struct nexthop *nh,
                          struct nh_info *nhi, struct nh_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct fib_nh *fib_nh = &nhi->fib_nh;
        struct fib_config fib_cfg = {
                .fc_oif   = cfg->nh_ifindex,
                .fc_gw4   = cfg->gw.ipv4,
                .fc_gw_family = cfg->gw.ipv4 ? AF_INET : 0,
                .fc_flags = cfg->nh_flags,
                .fc_nlinfo = cfg->nlinfo,
                .fc_encap = cfg->nh_encap,
                .fc_encap_type = cfg->nh_encap_type,
        };
        u32 tb_id = (cfg->dev ? l3mdev_fib_table(cfg->dev) : RT_TABLE_MAIN);
        int err;

        err = fib_nh_init(net, fib_nh, &fib_cfg, 1, extack);
        if (err) {
                fib_nh_release(net, fib_nh);
                goto out;
        }

        if (nhi->fdb_nh)
                goto out;

        /* sets nh_dev if successful */
        err = fib_check_nh(net, fib_nh, tb_id, 0, extack);
        if (!err) {
                nh->nh_flags = fib_nh->fib_nh_flags;
                fib_info_update_nhc_saddr(net, &fib_nh->nh_common,
                                          !fib_nh->fib_nh_scope ? 0 : fib_nh->fib_nh_scope - 1);
        } else {
                fib_nh_release(net, fib_nh);
        }
out:
        return err;
}

static int nh_create_ipv6(struct net *net,  struct nexthop *nh,
                          struct nh_info *nhi, struct nh_config *cfg,
                          struct netlink_ext_ack *extack)
{
        struct fib6_nh *fib6_nh = &nhi->fib6_nh;
        struct fib6_config fib6_cfg = {
                .fc_table = l3mdev_fib_table(cfg->dev),
                .fc_ifindex = cfg->nh_ifindex,
                .fc_gateway = cfg->gw.ipv6,
                .fc_flags = cfg->nh_flags,
                .fc_nlinfo = cfg->nlinfo,
                .fc_encap = cfg->nh_encap,
                .fc_encap_type = cfg->nh_encap_type,
                .fc_is_fdb = cfg->nh_fdb,
        };
        int err;

        if (!ipv6_addr_any(&cfg->gw.ipv6))
                fib6_cfg.fc_flags |= RTF_GATEWAY;

        /* sets nh_dev if successful */
        err = ipv6_stub->fib6_nh_init(net, fib6_nh, &fib6_cfg, GFP_KERNEL,
                                      extack);
        if (err) {
                /* IPv6 is not enabled, don't call fib6_nh_release */
                if (err == -EAFNOSUPPORT)
                        goto out;
                ipv6_stub->fib6_nh_release(fib6_nh);
        } else {
                nh->nh_flags = fib6_nh->fib_nh_flags;
        }
out:
        return err;
}

static struct nexthop *nexthop_create(struct net *net, struct nh_config *cfg,
                                      struct netlink_ext_ack *extack)
{
        struct nh_info *nhi;
        struct nexthop *nh;
        int err = 0;

        nh = nexthop_alloc();
        if (!nh)
                return ERR_PTR(-ENOMEM);

        nhi = kzalloc(sizeof(*nhi), GFP_KERNEL);
        if (!nhi) {
                kfree(nh);
                return ERR_PTR(-ENOMEM);
        }

        nh->nh_flags = cfg->nh_flags;
        nh->net = net;

        nhi->nh_parent = nh;
        nhi->family = cfg->nh_family;
        nhi->fib_nhc.nhc_scope = RT_SCOPE_LINK;

        if (cfg->nh_fdb)
                nhi->fdb_nh = 1;

        if (cfg->nh_blackhole) {
                nhi->reject_nh = 1;
                cfg->nh_ifindex = net->loopback_dev->ifindex;
        }

        switch (cfg->nh_family) {
        case AF_INET:
                err = nh_create_ipv4(net, nh, nhi, cfg, extack);
                break;
        case AF_INET6:
                err = nh_create_ipv6(net, nh, nhi, cfg, extack);
                break;
        }

        if (err) {
                kfree(nhi);
                kfree(nh);
                return ERR_PTR(err);
        }

        /* add the entry to the device based hash */
        if (!nhi->fdb_nh)
                nexthop_devhash_add(net, nhi);

        rcu_assign_pointer(nh->nh_info, nhi);

        return nh;
}

/* called with rtnl lock held */
static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
                                   struct netlink_ext_ack *extack)
{
        struct nexthop *nh;
        int err;

        if (!cfg->nh_id) {
                cfg->nh_id = nh_find_unused_id(net);
                if (!cfg->nh_id) {
                        NL_SET_ERR_MSG(extack, "No unused id");
                        return ERR_PTR(-EINVAL);
                }
        }

        if (cfg->nh_grp)
                nh = nexthop_create_group(net, cfg);
        else
                nh = nexthop_create(net, cfg, extack);

        if (IS_ERR(nh))
                return nh;

        refcount_set(&nh->refcnt, 1);
        nh->id = cfg->nh_id;
        nh->protocol = cfg->nh_protocol;
        nh->net = net;

        err = insert_nexthop(net, nh, cfg, extack);
        if (err) {
                __remove_nexthop(net, nh, NULL);
                nexthop_put(nh);
                nh = ERR_PTR(err);
        }

        return nh;
}

static int rtm_nh_get_timer(struct nlattr *attr, unsigned long fallback,
                            unsigned long *timer_p, bool *has_p,
                            struct netlink_ext_ack *extack)
{
        unsigned long timer;
        u32 value;

        if (!attr) {
                *timer_p = fallback;
                *has_p = false;
                return 0;
        }

        value = nla_get_u32(attr);
        timer = clock_t_to_jiffies(value);
        if (timer == ~0UL) {
                NL_SET_ERR_MSG(extack, "Timer value too large");
                return -EINVAL;
        }

        *timer_p = timer;
        *has_p = true;
        return 0;
}

static int rtm_to_nh_config_grp_res(struct nlattr *res, struct nh_config *cfg,
                                    struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_policy_new)] = {};
        int err;

        if (res) {
                err = nla_parse_nested(tb,
                                       ARRAY_SIZE(rtm_nh_res_policy_new) - 1,
                                       res, rtm_nh_res_policy_new, extack);
                if (err < 0)
                        return err;
        }

        if (tb[NHA_RES_GROUP_BUCKETS]) {
                cfg->nh_grp_res_num_buckets =
                        nla_get_u16(tb[NHA_RES_GROUP_BUCKETS]);
                cfg->nh_grp_res_has_num_buckets = true;
                if (!cfg->nh_grp_res_num_buckets) {
                        NL_SET_ERR_MSG(extack, "Number of buckets needs to be non-0");
                        return -EINVAL;
                }
        }

        err = rtm_nh_get_timer(tb[NHA_RES_GROUP_IDLE_TIMER],
                               NH_RES_DEFAULT_IDLE_TIMER,
                               &cfg->nh_grp_res_idle_timer,
                               &cfg->nh_grp_res_has_idle_timer,
                               extack);
        if (err)
                return err;

        return rtm_nh_get_timer(tb[NHA_RES_GROUP_UNBALANCED_TIMER],
                                NH_RES_DEFAULT_UNBALANCED_TIMER,
                                &cfg->nh_grp_res_unbalanced_timer,
                                &cfg->nh_grp_res_has_unbalanced_timer,
                                extack);
}

static int rtm_to_nh_config(struct net *net, struct sk_buff *skb,
                            struct nlmsghdr *nlh, struct nlattr **tb,
                            struct nh_config *cfg,
                            struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm = nlmsg_data(nlh);
        int err;

        err = -EINVAL;
        if (nhm->resvd || nhm->nh_scope) {
                NL_SET_ERR_MSG(extack, "Invalid values in ancillary header");
                goto out;
        }
        if (nhm->nh_flags & ~NEXTHOP_VALID_USER_FLAGS) {
                NL_SET_ERR_MSG(extack, "Invalid nexthop flags in ancillary header");
                goto out;
        }

        switch (nhm->nh_family) {
        case AF_INET:
        case AF_INET6:
                break;
        case AF_UNSPEC:
                if (tb[NHA_GROUP])
                        break;
                fallthrough;
        default:
                NL_SET_ERR_MSG(extack, "Invalid address family");
                goto out;
        }

        memset(cfg, 0, sizeof(*cfg));
        cfg->nlflags = nlh->nlmsg_flags;
        cfg->nlinfo.portid = NETLINK_CB(skb).portid;
        cfg->nlinfo.nlh = nlh;
        cfg->nlinfo.nl_net = net;

        cfg->nh_family = nhm->nh_family;
        cfg->nh_protocol = nhm->nh_protocol;
        cfg->nh_flags = nhm->nh_flags;

        if (tb[NHA_ID])
                cfg->nh_id = nla_get_u32(tb[NHA_ID]);

        if (tb[NHA_FDB]) {
                if (tb[NHA_OIF] || tb[NHA_BLACKHOLE] ||
                    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE]) {
                        NL_SET_ERR_MSG(extack, "Fdb attribute can not be used with encap, oif or blackhole");
                        goto out;
                }
                if (nhm->nh_flags) {
                        NL_SET_ERR_MSG(extack, "Unsupported nexthop flags in ancillary header");
                        goto out;
                }
                cfg->nh_fdb = nla_get_flag(tb[NHA_FDB]);
        }

        if (tb[NHA_GROUP]) {
                if (nhm->nh_family != AF_UNSPEC) {
                        NL_SET_ERR_MSG(extack, "Invalid family for group");
                        goto out;
                }
                cfg->nh_grp = tb[NHA_GROUP];

                cfg->nh_grp_type = NEXTHOP_GRP_TYPE_MPATH;
                if (tb[NHA_GROUP_TYPE])
                        cfg->nh_grp_type = nla_get_u16(tb[NHA_GROUP_TYPE]);

                if (cfg->nh_grp_type > NEXTHOP_GRP_TYPE_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid group type");
                        goto out;
                }

                err = nh_check_attr_group(net, tb, ARRAY_SIZE(rtm_nh_policy_new),
                                          cfg->nh_grp_type, extack);
                if (err)
                        goto out;

                if (cfg->nh_grp_type == NEXTHOP_GRP_TYPE_RES)
                        err = rtm_to_nh_config_grp_res(tb[NHA_RES_GROUP],
                                                       cfg, extack);

                if (tb[NHA_HW_STATS_ENABLE])
                        cfg->nh_hw_stats = nla_get_u32(tb[NHA_HW_STATS_ENABLE]);

                /* no other attributes should be set */
                goto out;
        }

        if (tb[NHA_BLACKHOLE]) {
                if (tb[NHA_GATEWAY] || tb[NHA_OIF] ||
                    tb[NHA_ENCAP]   || tb[NHA_ENCAP_TYPE] || tb[NHA_FDB]) {
                        NL_SET_ERR_MSG(extack, "Blackhole attribute can not be used with gateway, oif, encap or fdb");
                        goto out;
                }

                cfg->nh_blackhole = 1;
                err = 0;
                goto out;
        }

        if (!cfg->nh_fdb && !tb[NHA_OIF]) {
                NL_SET_ERR_MSG(extack, "Device attribute required for non-blackhole and non-fdb nexthops");
                goto out;
        }

        err = -EINVAL;
        if (tb[NHA_GATEWAY]) {
                struct nlattr *gwa = tb[NHA_GATEWAY];

                switch (cfg->nh_family) {
                case AF_INET:
                        if (nla_len(gwa) != sizeof(u32)) {
                                NL_SET_ERR_MSG(extack, "Invalid gateway");
                                goto out;
                        }
                        cfg->gw.ipv4 = nla_get_be32(gwa);
                        break;
                case AF_INET6:
                        if (nla_len(gwa) != sizeof(struct in6_addr)) {
                                NL_SET_ERR_MSG(extack, "Invalid gateway");
                                goto out;
                        }
                        cfg->gw.ipv6 = nla_get_in6_addr(gwa);
                        break;
                default:
                        NL_SET_ERR_MSG(extack,
                                       "Unknown address family for gateway");
                        goto out;
                }
        } else {
                /* device only nexthop (no gateway) */
                if (cfg->nh_flags & RTNH_F_ONLINK) {
                        NL_SET_ERR_MSG(extack,
                                       "ONLINK flag can not be set for nexthop without a gateway");
                        goto out;
                }
        }

        if (tb[NHA_ENCAP]) {
                cfg->nh_encap = tb[NHA_ENCAP];

                if (!tb[NHA_ENCAP_TYPE]) {
                        NL_SET_ERR_MSG(extack, "LWT encapsulation type is missing");
                        goto out;
                }

                cfg->nh_encap_type = nla_get_u16(tb[NHA_ENCAP_TYPE]);
                err = lwtunnel_valid_encap_type(cfg->nh_encap_type, extack);
                if (err < 0)
                        goto out;

        } else if (tb[NHA_ENCAP_TYPE]) {
                NL_SET_ERR_MSG(extack, "LWT encapsulation attribute is missing");
                goto out;
        }

        if (tb[NHA_HW_STATS_ENABLE]) {
                NL_SET_ERR_MSG(extack, "Cannot enable nexthop hardware statistics for non-group nexthops");
                goto out;
        }

        err = 0;
out:
        return err;
}

static int rtm_to_nh_config_rtnl(struct net *net, struct nlattr **tb,
                                 struct nh_config *cfg,
                                 struct netlink_ext_ack *extack)
{
        if (tb[NHA_GROUP])
                return nh_check_attr_group_rtnl(net, tb, extack);

        if (tb[NHA_OIF]) {
                cfg->nh_ifindex = nla_get_u32(tb[NHA_OIF]);
                if (cfg->nh_ifindex)
                        cfg->dev = __dev_get_by_index(net, cfg->nh_ifindex);

                if (!cfg->dev) {
                        NL_SET_ERR_MSG(extack, "Invalid device index");
                        return -EINVAL;
                }

                if (!(cfg->dev->flags & IFF_UP)) {
                        NL_SET_ERR_MSG(extack, "Nexthop device is not up");
                        return -ENETDOWN;
                }

                if (!netif_carrier_ok(cfg->dev)) {
                        NL_SET_ERR_MSG(extack, "Carrier for nexthop device is down");
                        return -ENETDOWN;
                }
        }

        return 0;
}

/* rtnl */
static int rtm_new_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_new)];
        struct net *net = sock_net(skb->sk);
        struct nh_config cfg;
        struct nexthop *nh;
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_new) - 1,
                          rtm_nh_policy_new, extack);
        if (err < 0)
                goto out;

        err = rtm_to_nh_config(net, skb, nlh, tb, &cfg, extack);
        if (err)
                goto out;

        if (cfg.nlflags & NLM_F_REPLACE && !cfg.nh_id) {
                NL_SET_ERR_MSG(extack, "Replace requires nexthop id");
                err = -EINVAL;
                goto out;
        }

        rtnl_net_lock(net);

        err = rtm_to_nh_config_rtnl(net, tb, &cfg, extack);
        if (err)
                goto unlock;

        nh = nexthop_add(net, &cfg, extack);
        if (IS_ERR(nh))
                err = PTR_ERR(nh);

unlock:
        rtnl_net_unlock(net);
out:
        return err;
}

static int nh_valid_get_del_req(const struct nlmsghdr *nlh,
                                struct nlattr **tb, u32 *id, u32 *op_flags,
                                struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm = nlmsg_data(nlh);

        if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
                NL_SET_ERR_MSG(extack, "Invalid values in header");
                return -EINVAL;
        }

        if (!tb[NHA_ID]) {
                NL_SET_ERR_MSG(extack, "Nexthop id is missing");
                return -EINVAL;
        }

        *id = nla_get_u32(tb[NHA_ID]);
        if (!(*id)) {
                NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                return -EINVAL;
        }

        if (op_flags)
                *op_flags = nla_get_u32_default(tb[NHA_OP_FLAGS], 0);

        return 0;
}

/* rtnl */
static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_del)];
        struct net *net = sock_net(skb->sk);
        struct nl_info nlinfo = {
                .nlh = nlh,
                .nl_net = net,
                .portid = NETLINK_CB(skb).portid,
        };
        struct nexthop *nh;
        int err;
        u32 id;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_del) - 1, rtm_nh_policy_del,
                          extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, &id, NULL, extack);
        if (err)
                return err;

        rtnl_net_lock(net);

        nh = nexthop_find_by_id(net, id);
        if (nh)
                remove_nexthop(net, nh, &nlinfo);
        else
                err = -ENOENT;

        rtnl_net_unlock(net);

        return err;
}

/* rtnl */
static int rtm_get_nexthop(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                           struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get)];
        struct net *net = sock_net(in_skb->sk);
        struct sk_buff *skb = NULL;
        struct nexthop *nh;
        u32 op_flags;
        int err;
        u32 id;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_get) - 1, rtm_nh_policy_get,
                          extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, &id, &op_flags, extack);
        if (err)
                return err;

        err = -ENOBUFS;
        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                goto out;

        err = -ENOENT;
        nh = nexthop_find_by_id(net, id);
        if (!nh)
                goto errout_free;

        err = nh_fill_node(skb, nh, RTM_NEWNEXTHOP, NETLINK_CB(in_skb).portid,
                           nlh->nlmsg_seq, 0, op_flags);
        if (err < 0) {
                WARN_ON(err == -EMSGSIZE);
                goto errout_free;
        }

        err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
out:
        return err;
errout_free:
        kfree_skb(skb);
        goto out;
}

struct nh_dump_filter {
        u32 nh_id;
        int dev_idx;
        int master_idx;
        bool group_filter;
        bool fdb_filter;
        u32 res_bucket_nh_id;
        u32 op_flags;
};

static bool nh_dump_filtered(struct nexthop *nh,
                             struct nh_dump_filter *filter, u8 family)
{
        const struct net_device *dev;
        const struct nh_info *nhi;

        if (filter->group_filter && !nh->is_group)
                return true;

        if (!filter->dev_idx && !filter->master_idx && !family)
                return false;

        if (nh->is_group)
                return true;

        nhi = rtnl_dereference(nh->nh_info);
        if (family && nhi->family != family)
                return true;

        dev = nhi->fib_nhc.nhc_dev;
        if (filter->dev_idx && (!dev || dev->ifindex != filter->dev_idx))
                return true;

        if (filter->master_idx) {
                struct net_device *master;

                if (!dev)
                        return true;

                master = netdev_master_upper_dev_get((struct net_device *)dev);
                if (!master || master->ifindex != filter->master_idx)
                        return true;
        }

        return false;
}

static int __nh_valid_dump_req(const struct nlmsghdr *nlh, struct nlattr **tb,
                               struct nh_dump_filter *filter,
                               struct netlink_ext_ack *extack)
{
        struct nhmsg *nhm;
        u32 idx;

        if (tb[NHA_OIF]) {
                idx = nla_get_u32(tb[NHA_OIF]);
                if (idx > INT_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid device index");
                        return -EINVAL;
                }
                filter->dev_idx = idx;
        }
        if (tb[NHA_MASTER]) {
                idx = nla_get_u32(tb[NHA_MASTER]);
                if (idx > INT_MAX) {
                        NL_SET_ERR_MSG(extack, "Invalid master device index");
                        return -EINVAL;
                }
                filter->master_idx = idx;
        }
        filter->group_filter = nla_get_flag(tb[NHA_GROUPS]);
        filter->fdb_filter = nla_get_flag(tb[NHA_FDB]);

        nhm = nlmsg_data(nlh);
        if (nhm->nh_protocol || nhm->resvd || nhm->nh_scope || nhm->nh_flags) {
                NL_SET_ERR_MSG(extack, "Invalid values in header for nexthop dump request");
                return -EINVAL;
        }

        return 0;
}

static int nh_valid_dump_req(const struct nlmsghdr *nlh,
                             struct nh_dump_filter *filter,
                             struct netlink_callback *cb)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_dump) - 1,
                          rtm_nh_policy_dump, cb->extack);
        if (err < 0)
                return err;

        filter->op_flags = nla_get_u32_default(tb[NHA_OP_FLAGS], 0);

        return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
}

struct rtm_dump_nh_ctx {
        u32 idx;
};

static struct rtm_dump_nh_ctx *
rtm_dump_nh_ctx(struct netlink_callback *cb)
{
        struct rtm_dump_nh_ctx *ctx = (void *)cb->ctx;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
        return ctx;
}

static int rtm_dump_walk_nexthops(struct sk_buff *skb,
                                  struct netlink_callback *cb,
                                  struct rb_root *root,
                                  struct rtm_dump_nh_ctx *ctx,
                                  int (*nh_cb)(struct sk_buff *skb,
                                               struct netlink_callback *cb,
                                               struct nexthop *nh, void *data),
                                  void *data)
{
        struct rb_node *node;
        int s_idx;
        int err;

        s_idx = ctx->idx;
        for (node = rb_first(root); node; node = rb_next(node)) {
                struct nexthop *nh;

                nh = rb_entry(node, struct nexthop, rb_node);
                if (nh->id < s_idx)
                        continue;

                ctx->idx = nh->id;
                err = nh_cb(skb, cb, nh, data);
                if (err)
                        return err;
        }

        return 0;
}

static int rtm_dump_nexthop_cb(struct sk_buff *skb, struct netlink_callback *cb,
                               struct nexthop *nh, void *data)
{
        struct nhmsg *nhm = nlmsg_data(cb->nlh);
        struct nh_dump_filter *filter = data;

        if (nh_dump_filtered(nh, filter, nhm->nh_family))
                return 0;

        return nh_fill_node(skb, nh, RTM_NEWNEXTHOP,
                            NETLINK_CB(cb->skb).portid,
                            cb->nlh->nlmsg_seq, NLM_F_MULTI, filter->op_flags);
}

/* rtnl */
static int rtm_dump_nexthop(struct sk_buff *skb, struct netlink_callback *cb)
{
        struct rtm_dump_nh_ctx *ctx = rtm_dump_nh_ctx(cb);
        struct net *net = sock_net(skb->sk);
        struct rb_root *root = &net->nexthop.rb_root;
        struct nh_dump_filter filter = {};
        int err;

        err = nh_valid_dump_req(cb->nlh, &filter, cb);
        if (err < 0)
                return err;

        err = rtm_dump_walk_nexthops(skb, cb, root, ctx,
                                     &rtm_dump_nexthop_cb, &filter);

        cb->seq = net->nexthop.seq;
        nl_dump_check_consistent(cb, nlmsg_hdr(skb));
        return err;
}

static struct nexthop *
nexthop_find_group_resilient(struct net *net, u32 id,
                             struct netlink_ext_ack *extack)
{
        struct nh_group *nhg;
        struct nexthop *nh;

        nh = nexthop_find_by_id(net, id);
        if (!nh)
                return ERR_PTR(-ENOENT);

        if (!nh->is_group) {
                NL_SET_ERR_MSG(extack, "Not a nexthop group");
                return ERR_PTR(-EINVAL);
        }

        nhg = rtnl_dereference(nh->nh_grp);
        if (!nhg->resilient) {
                NL_SET_ERR_MSG(extack, "Nexthop group not of type resilient");
                return ERR_PTR(-EINVAL);
        }

        return nh;
}

static int nh_valid_dump_nhid(struct nlattr *attr, u32 *nh_id_p,
                              struct netlink_ext_ack *extack)
{
        u32 idx;

        if (attr) {
                idx = nla_get_u32(attr);
                if (!idx) {
                        NL_SET_ERR_MSG(extack, "Invalid nexthop id");
                        return -EINVAL;
                }
                *nh_id_p = idx;
        } else {
                *nh_id_p = 0;
        }

        return 0;
}

static int nh_valid_dump_bucket_req(const struct nlmsghdr *nlh,
                                    struct nh_dump_filter *filter,
                                    struct netlink_callback *cb)
{
        struct nlattr *res_tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_dump)];
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_dump_bucket)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_dump_bucket) - 1,
                          rtm_nh_policy_dump_bucket, NULL);
        if (err < 0)
                return err;

        err = nh_valid_dump_nhid(tb[NHA_ID], &filter->nh_id, cb->extack);
        if (err)
                return err;

        if (tb[NHA_RES_BUCKET]) {
                size_t max = ARRAY_SIZE(rtm_nh_res_bucket_policy_dump) - 1;

                err = nla_parse_nested(res_tb, max,
                                       tb[NHA_RES_BUCKET],
                                       rtm_nh_res_bucket_policy_dump,
                                       cb->extack);
                if (err < 0)
                        return err;

                err = nh_valid_dump_nhid(res_tb[NHA_RES_BUCKET_NH_ID],
                                         &filter->res_bucket_nh_id,
                                         cb->extack);
                if (err)
                        return err;
        }

        return __nh_valid_dump_req(nlh, tb, filter, cb->extack);
}

struct rtm_dump_res_bucket_ctx {
        struct rtm_dump_nh_ctx nh;
        u16 bucket_index;
};

static struct rtm_dump_res_bucket_ctx *
rtm_dump_res_bucket_ctx(struct netlink_callback *cb)
{
        struct rtm_dump_res_bucket_ctx *ctx = (void *)cb->ctx;

        BUILD_BUG_ON(sizeof(*ctx) > sizeof(cb->ctx));
        return ctx;
}

struct rtm_dump_nexthop_bucket_data {
        struct rtm_dump_res_bucket_ctx *ctx;
        struct nh_dump_filter filter;
};

static int rtm_dump_nexthop_bucket_nh(struct sk_buff *skb,
                                      struct netlink_callback *cb,
                                      struct nexthop *nh,
                                      struct rtm_dump_nexthop_bucket_data *dd)
{
        u32 portid = NETLINK_CB(cb->skb).portid;
        struct nhmsg *nhm = nlmsg_data(cb->nlh);
        struct nh_res_table *res_table;
        struct nh_group *nhg;
        u16 bucket_index;
        int err;

        nhg = rtnl_dereference(nh->nh_grp);
        res_table = rtnl_dereference(nhg->res_table);
        for (bucket_index = dd->ctx->bucket_index;
             bucket_index < res_table->num_nh_buckets;
             bucket_index++) {
                struct nh_res_bucket *bucket;
                struct nh_grp_entry *nhge;

                bucket = &res_table->nh_buckets[bucket_index];
                nhge = rtnl_dereference(bucket->nh_entry);
                if (nh_dump_filtered(nhge->nh, &dd->filter, nhm->nh_family))
                        continue;

                if (dd->filter.res_bucket_nh_id &&
                    dd->filter.res_bucket_nh_id != nhge->nh->id)
                        continue;

                dd->ctx->bucket_index = bucket_index;
                err = nh_fill_res_bucket(skb, nh, bucket, bucket_index,
                                         RTM_NEWNEXTHOPBUCKET, portid,
                                         cb->nlh->nlmsg_seq, NLM_F_MULTI,
                                         cb->extack);
                if (err)
                        return err;
        }

        dd->ctx->bucket_index = 0;

        return 0;
}

static int rtm_dump_nexthop_bucket_cb(struct sk_buff *skb,
                                      struct netlink_callback *cb,
                                      struct nexthop *nh, void *data)
{
        struct rtm_dump_nexthop_bucket_data *dd = data;
        struct nh_group *nhg;

        if (!nh->is_group)
                return 0;

        nhg = rtnl_dereference(nh->nh_grp);
        if (!nhg->resilient)
                return 0;

        return rtm_dump_nexthop_bucket_nh(skb, cb, nh, dd);
}

/* rtnl */
static int rtm_dump_nexthop_bucket(struct sk_buff *skb,
                                   struct netlink_callback *cb)
{
        struct rtm_dump_res_bucket_ctx *ctx = rtm_dump_res_bucket_ctx(cb);
        struct rtm_dump_nexthop_bucket_data dd = { .ctx = ctx };
        struct net *net = sock_net(skb->sk);
        struct nexthop *nh;
        int err;

        err = nh_valid_dump_bucket_req(cb->nlh, &dd.filter, cb);
        if (err)
                return err;

        if (dd.filter.nh_id) {
                nh = nexthop_find_group_resilient(net, dd.filter.nh_id,
                                                  cb->extack);
                if (IS_ERR(nh))
                        return PTR_ERR(nh);
                err = rtm_dump_nexthop_bucket_nh(skb, cb, nh, &dd);
        } else {
                struct rb_root *root = &net->nexthop.rb_root;

                err = rtm_dump_walk_nexthops(skb, cb, root, &ctx->nh,
                                             &rtm_dump_nexthop_bucket_cb, &dd);
        }

        cb->seq = net->nexthop.seq;
        nl_dump_check_consistent(cb, nlmsg_hdr(skb));
        return err;
}

static int nh_valid_get_bucket_req_res_bucket(struct nlattr *res,
                                              u16 *bucket_index,
                                              struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_res_bucket_policy_get)];
        int err;

        err = nla_parse_nested(tb, ARRAY_SIZE(rtm_nh_res_bucket_policy_get) - 1,
                               res, rtm_nh_res_bucket_policy_get, extack);
        if (err < 0)
                return err;

        if (!tb[NHA_RES_BUCKET_INDEX]) {
                NL_SET_ERR_MSG(extack, "Bucket index is missing");
                return -EINVAL;
        }

        *bucket_index = nla_get_u16(tb[NHA_RES_BUCKET_INDEX]);
        return 0;
}

static int nh_valid_get_bucket_req(const struct nlmsghdr *nlh,
                                   u32 *id, u16 *bucket_index,
                                   struct netlink_ext_ack *extack)
{
        struct nlattr *tb[ARRAY_SIZE(rtm_nh_policy_get_bucket)];
        int err;

        err = nlmsg_parse(nlh, sizeof(struct nhmsg), tb,
                          ARRAY_SIZE(rtm_nh_policy_get_bucket) - 1,
                          rtm_nh_policy_get_bucket, extack);
        if (err < 0)
                return err;

        err = nh_valid_get_del_req(nlh, tb, id, NULL, extack);
        if (err)
                return err;

        if (!tb[NHA_RES_BUCKET]) {
                NL_SET_ERR_MSG(extack, "Bucket information is missing");
                return -EINVAL;
        }

        err = nh_valid_get_bucket_req_res_bucket(tb[NHA_RES_BUCKET],
                                                 bucket_index, extack);
        if (err)
                return err;

        return 0;
}

/* rtnl */
static int rtm_get_nexthop_bucket(struct sk_buff *in_skb, struct nlmsghdr *nlh,
                                  struct netlink_ext_ack *extack)
{
        struct net *net = sock_net(in_skb->sk);
        struct nh_res_table *res_table;
        struct sk_buff *skb = NULL;
        struct nh_group *nhg;
        struct nexthop *nh;
        u16 bucket_index;
        int err;
        u32 id;

        err = nh_valid_get_bucket_req(nlh, &id, &bucket_index, extack);
        if (err)
                return err;

        nh = nexthop_find_group_resilient(net, id, extack);
        if (IS_ERR(nh))
                return PTR_ERR(nh);

        nhg = rtnl_dereference(nh->nh_grp);
        res_table = rtnl_dereference(nhg->res_table);
        if (bucket_index >= res_table->num_nh_buckets) {
                NL_SET_ERR_MSG(extack, "Bucket index out of bounds");
                return -ENOENT;
        }

        skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
        if (!skb)
                return -ENOBUFS;

        err = nh_fill_res_bucket(skb, nh, &res_table->nh_buckets[bucket_index],
                                 bucket_index, RTM_NEWNEXTHOPBUCKET,
                                 NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
                                 0, extack);
        if (err < 0) {
                WARN_ON(err == -EMSGSIZE);
                goto errout_free;
        }

        return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);

errout_free:
        kfree_skb(skb);
        return err;
}

static void nexthop_sync_mtu(struct net_device *dev, u32 orig_mtu)
{
        unsigned int hash = nh_dev_hashfn(dev->ifindex);
        struct net *net = dev_net(dev);
        struct hlist_head *head = &net->nexthop.devhash[hash];
        struct hlist_node *n;
        struct nh_info *nhi;

        hlist_for_each_entry_safe(nhi, n, head, dev_hash) {
                if (nhi->fib_nhc.nhc_dev == dev) {
                        if (nhi->family == AF_INET)
                                fib_nhc_update_mtu(&nhi->fib_nhc, dev->mtu,
                                                   orig_mtu);
                }
        }
}

/* rtnl */
static int nh_netdev_event(struct notifier_block *this,
                           unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct netdev_notifier_info_ext *info_ext;

        switch (event) {
        case NETDEV_DOWN:
        case NETDEV_UNREGISTER:
                nexthop_flush_dev(dev, event);
                break;
        case NETDEV_CHANGE:
                if (!(dev_get_flags(dev) & (IFF_RUNNING | IFF_LOWER_UP)))
                        nexthop_flush_dev(dev, event);
                break;
        case NETDEV_CHANGEMTU:
                info_ext = ptr;
                nexthop_sync_mtu(dev, info_ext->ext.mtu);
                rt_cache_flush(dev_net(dev));
                break;
        }
        return NOTIFY_DONE;
}

static struct notifier_block nh_netdev_notifier = {
        .notifier_call = nh_netdev_event,
};

static int nexthops_dump(struct net *net, struct notifier_block *nb,
                         enum nexthop_event_type event_type,
                         struct netlink_ext_ack *extack)
{
        struct rb_root *root = &net->nexthop.rb_root;
        struct rb_node *node;
        int err = 0;

        for (node = rb_first(root); node; node = rb_next(node)) {
                struct nexthop *nh;

                nh = rb_entry(node, struct nexthop, rb_node);
                err = call_nexthop_notifier(nb, net, event_type, nh, extack);
                if (err)
                        break;
        }

        return err;
}

int register_nexthop_notifier(struct net *net, struct notifier_block *nb,
                              struct netlink_ext_ack *extack)
{
        int err;

        rtnl_lock();
        err = nexthops_dump(net, nb, NEXTHOP_EVENT_REPLACE, extack);
        if (err)
                goto unlock;
        err = blocking_notifier_chain_register(&net->nexthop.notifier_chain,
                                               nb);
unlock:
        rtnl_unlock();
        return err;
}
EXPORT_SYMBOL(register_nexthop_notifier);

int __unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{
        int err;

        err = blocking_notifier_chain_unregister(&net->nexthop.notifier_chain,
                                                 nb);
        if (!err)
                nexthops_dump(net, nb, NEXTHOP_EVENT_DEL, NULL);
        return err;
}
EXPORT_SYMBOL(__unregister_nexthop_notifier);

int unregister_nexthop_notifier(struct net *net, struct notifier_block *nb)
{
        int err;

        rtnl_lock();
        err = __unregister_nexthop_notifier(net, nb);
        rtnl_unlock();
        return err;
}
EXPORT_SYMBOL(unregister_nexthop_notifier);

void nexthop_set_hw_flags(struct net *net, u32 id, bool offload, bool trap)
{
        struct nexthop *nexthop;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop)
                goto out;

        nexthop->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
        if (offload)
                nexthop->nh_flags |= RTNH_F_OFFLOAD;
        if (trap)
                nexthop->nh_flags |= RTNH_F_TRAP;

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_set_hw_flags);

void nexthop_bucket_set_hw_flags(struct net *net, u32 id, u16 bucket_index,
                                 bool offload, bool trap)
{
        struct nh_res_table *res_table;
        struct nh_res_bucket *bucket;
        struct nexthop *nexthop;
        struct nh_group *nhg;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop || !nexthop->is_group)
                goto out;

        nhg = rcu_dereference(nexthop->nh_grp);
        if (!nhg->resilient)
                goto out;

        if (bucket_index >= nhg->res_table->num_nh_buckets)
                goto out;

        res_table = rcu_dereference(nhg->res_table);
        bucket = &res_table->nh_buckets[bucket_index];
        bucket->nh_flags &= ~(RTNH_F_OFFLOAD | RTNH_F_TRAP);
        if (offload)
                bucket->nh_flags |= RTNH_F_OFFLOAD;
        if (trap)
                bucket->nh_flags |= RTNH_F_TRAP;

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_bucket_set_hw_flags);

void nexthop_res_grp_activity_update(struct net *net, u32 id, u16 num_buckets,
                                     unsigned long *activity)
{
        struct nh_res_table *res_table;
        struct nexthop *nexthop;
        struct nh_group *nhg;
        u16 i;

        rcu_read_lock();

        nexthop = nexthop_find_by_id(net, id);
        if (!nexthop || !nexthop->is_group)
                goto out;

        nhg = rcu_dereference(nexthop->nh_grp);
        if (!nhg->resilient)
                goto out;

        /* Instead of silently ignoring some buckets, demand that the sizes
         * be the same.
         */
        res_table = rcu_dereference(nhg->res_table);
        if (num_buckets != res_table->num_nh_buckets)
                goto out;

        for (i = 0; i < num_buckets; i++) {
                if (test_bit(i, activity))
                        nh_res_bucket_set_busy(&res_table->nh_buckets[i]);
        }

out:
        rcu_read_unlock();
}
EXPORT_SYMBOL(nexthop_res_grp_activity_update);

static void __net_exit nexthop_net_exit_rtnl(struct net *net,
                                             struct list_head *dev_to_kill)
{
        ASSERT_RTNL_NET(net);
        flush_all_nexthops(net);
}

static void __net_exit nexthop_net_exit(struct net *net)
{
        kfree(net->nexthop.devhash);
        net->nexthop.devhash = NULL;
}

static int __net_init nexthop_net_init(struct net *net)
{
        size_t sz = sizeof(struct hlist_head) * NH_DEV_HASHSIZE;

        net->nexthop.rb_root = RB_ROOT;
        net->nexthop.devhash = kzalloc(sz, GFP_KERNEL);
        if (!net->nexthop.devhash)
                return -ENOMEM;
        BLOCKING_INIT_NOTIFIER_HEAD(&net->nexthop.notifier_chain);

        return 0;
}

static struct pernet_operations nexthop_net_ops = {
        .init = nexthop_net_init,
        .exit = nexthop_net_exit,
        .exit_rtnl = nexthop_net_exit_rtnl,
};

static const struct rtnl_msg_handler nexthop_rtnl_msg_handlers[] __initconst = {
        {.msgtype = RTM_NEWNEXTHOP, .doit = rtm_new_nexthop,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_DELNEXTHOP, .doit = rtm_del_nexthop,
         .flags = RTNL_FLAG_DOIT_PERNET},
        {.msgtype = RTM_GETNEXTHOP, .doit = rtm_get_nexthop,
         .dumpit = rtm_dump_nexthop},
        {.msgtype = RTM_GETNEXTHOPBUCKET, .doit = rtm_get_nexthop_bucket,
         .dumpit = rtm_dump_nexthop_bucket},
        {.protocol = PF_INET, .msgtype = RTM_NEWNEXTHOP,
         .doit = rtm_new_nexthop, .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET, .msgtype = RTM_GETNEXTHOP,
         .dumpit = rtm_dump_nexthop},
        {.protocol = PF_INET6, .msgtype = RTM_NEWNEXTHOP,
         .doit = rtm_new_nexthop, .flags = RTNL_FLAG_DOIT_PERNET},
        {.protocol = PF_INET6, .msgtype = RTM_GETNEXTHOP,
         .dumpit = rtm_dump_nexthop},
};

static int __init nexthop_init(void)
{
        register_pernet_subsys(&nexthop_net_ops);

        register_netdevice_notifier(&nh_netdev_notifier);

        rtnl_register_many(nexthop_rtnl_msg_handlers);

        return 0;
}
subsys_initcall(nexthop_init);














































































































































































































































































































































































































































































































































































































































































































   29 


























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
// SPDX-License-Identifier: GPL-2.0
/*
 *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner
 *
 *  NOHZ implementation for low and high resolution timers
 *
 *  Started by: Thomas Gleixner and Ingo Molnar
 */
#include <linux/compiler.h>
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/percpu.h>
#include <linux/nmi.h>
#include <linux/profile.h>
#include <linux/sched/signal.h>
#include <linux/sched/clock.h>
#include <linux/sched/stat.h>
#include <linux/sched/nohz.h>
#include <linux/sched/loadavg.h>
#include <linux/module.h>
#include <linux/irq_work.h>
#include <linux/posix-timers.h>
#include <linux/context_tracking.h>
#include <linux/mm.h>

#include <asm/irq_regs.h>

#include "tick-internal.h"

#include <trace/events/timer.h>

/*
 * Per-CPU nohz control structure
 */
static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);

struct tick_sched *tick_get_tick_sched(int cpu)
{
        return &per_cpu(tick_cpu_sched, cpu);
}

/*
 * The time when the last jiffy update happened. Write access must hold
 * jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a
 * consistent view of jiffies and last_jiffies_update.
 */
static ktime_t last_jiffies_update;

/*
 * Must be called with interrupts disabled !
 */
static void tick_do_update_jiffies64(ktime_t now)
{
        unsigned long ticks = 1;
        ktime_t delta, nextp;

        /*
         * 64-bit can do a quick check without holding the jiffies lock and
         * without looking at the sequence count. The smp_load_acquire()
         * pairs with the update done later in this function.
         *
         * 32-bit cannot do that because the store of 'tick_next_period'
         * consists of two 32-bit stores, and the first store could be
         * moved by the CPU to a random point in the future.
         */
        if (IS_ENABLED(CONFIG_64BIT)) {
                if (ktime_before(now, smp_load_acquire(&tick_next_period)))
                        return;
        } else {
                unsigned int seq;

                /*
                 * Avoid contention on 'jiffies_lock' and protect the quick
                 * check with the sequence count.
                 */
                do {
                        seq = read_seqcount_begin(&jiffies_seq);
                        nextp = tick_next_period;
                } while (read_seqcount_retry(&jiffies_seq, seq));

                if (ktime_before(now, nextp))
                        return;
        }

        /* Quick check failed, i.e. update is required. */
        raw_spin_lock(&jiffies_lock);
        /*
         * Re-evaluate with the lock held. Another CPU might have done the
         * update already.
         */
        if (ktime_before(now, tick_next_period)) {
                raw_spin_unlock(&jiffies_lock);
                return;
        }

        write_seqcount_begin(&jiffies_seq);

        delta = ktime_sub(now, tick_next_period);
        if (unlikely(delta >= TICK_NSEC)) {
                /* Slow path for long idle sleep times */
                s64 incr = TICK_NSEC;

                ticks += ktime_divns(delta, incr);

                last_jiffies_update = ktime_add_ns(last_jiffies_update,
                                                   incr * ticks);
        } else {
                last_jiffies_update = ktime_add_ns(last_jiffies_update,
                                                   TICK_NSEC);
        }

        /* Advance jiffies to complete the 'jiffies_seq' protected job */
        jiffies_64 += ticks;

        /* Keep the tick_next_period variable up to date */
        nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC);

        if (IS_ENABLED(CONFIG_64BIT)) {
                /*
                 * Pairs with smp_load_acquire() in the lockless quick
                 * check above, and ensures that the update to 'jiffies_64' is
                 * not reordered vs. the store to 'tick_next_period', neither
                 * by the compiler nor by the CPU.
                 */
                smp_store_release(&tick_next_period, nextp);
        } else {
                /*
                 * A plain store is good enough on 32-bit, as the quick check
                 * above is protected by the sequence count.
                 */
                tick_next_period = nextp;
        }

        /*
         * Release the sequence count. calc_global_load() below is not
         * protected by it, but 'jiffies_lock' needs to be held to prevent
         * concurrent invocations.
         */
        write_seqcount_end(&jiffies_seq);

        calc_global_load();

        raw_spin_unlock(&jiffies_lock);
        update_wall_time();
}

/*
 * Initialize and return retrieve the jiffies update.
 */
static ktime_t tick_init_jiffy_update(void)
{
        ktime_t period;

        raw_spin_lock(&jiffies_lock);
        write_seqcount_begin(&jiffies_seq);

        /* Have we started the jiffies update yet ? */
        if (last_jiffies_update == 0) {
                u32 rem;

                /*
                 * Ensure that the tick is aligned to a multiple of
                 * TICK_NSEC.
                 */
                div_u64_rem(tick_next_period, TICK_NSEC, &rem);
                if (rem)
                        tick_next_period += TICK_NSEC - rem;

                last_jiffies_update = tick_next_period;
        }
        period = last_jiffies_update;

        write_seqcount_end(&jiffies_seq);
        raw_spin_unlock(&jiffies_lock);

        return period;
}

static inline int tick_sched_flag_test(struct tick_sched *ts,
                                       unsigned long flag)
{
        return !!(ts->flags & flag);
}

static inline void tick_sched_flag_set(struct tick_sched *ts,
                                       unsigned long flag)
{
        lockdep_assert_irqs_disabled();
        ts->flags |= flag;
}

static inline void tick_sched_flag_clear(struct tick_sched *ts,
                                         unsigned long flag)
{
        lockdep_assert_irqs_disabled();
        ts->flags &= ~flag;
}

#define MAX_STALLED_JIFFIES 5

static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
{
        int tick_cpu, cpu = smp_processor_id();

        /*
         * Check if the do_timer duty was dropped. We don't care about
         * concurrency: This happens only when the CPU in charge went
         * into a long sleep. If two CPUs happen to assign themselves to
         * this duty, then the jiffies update is still serialized by
         * 'jiffies_lock'.
         *
         * If nohz_full is enabled, this should not happen because the
         * 'tick_do_timer_cpu' CPU never relinquishes.
         */
        tick_cpu = READ_ONCE(tick_do_timer_cpu);

        if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && unlikely(tick_cpu == TICK_DO_TIMER_NONE)) {
#ifdef CONFIG_NO_HZ_FULL
                WARN_ON_ONCE(tick_nohz_full_running);
#endif
                WRITE_ONCE(tick_do_timer_cpu, cpu);
                tick_cpu = cpu;
        }

        /* Check if jiffies need an update */
        if (tick_cpu == cpu)
                tick_do_update_jiffies64(now);

        /*
         * If the jiffies update stalled for too long (timekeeper in stop_machine()
         * or VMEXIT'ed for several msecs), force an update.
         */
        if (ts->last_tick_jiffies != jiffies) {
                ts->stalled_jiffies = 0;
                ts->last_tick_jiffies = READ_ONCE(jiffies);
        } else {
                if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {
                        tick_do_update_jiffies64(now);
                        ts->stalled_jiffies = 0;
                        ts->last_tick_jiffies = READ_ONCE(jiffies);
                }
        }

        if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
                ts->got_idle_tick = 1;
}

static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
{
        /*
         * When we are idle and the tick is stopped, we have to touch
         * the watchdog as we might not schedule for a really long
         * time. This happens on completely idle SMP systems while
         * waiting on the login prompt. We also increment the "start of
         * idle" jiffy stamp so the idle accounting adjustment we do
         * when we go busy again does not account too many ticks.
         */
        if (IS_ENABLED(CONFIG_NO_HZ_COMMON) &&
            tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                touch_softlockup_watchdog_sched();
                if (is_idle_task(current))
                        ts->idle_jiffies++;
                /*
                 * In case the current tick fired too early past its expected
                 * expiration, make sure we don't bypass the next clock reprogramming
                 * to the same deadline.
                 */
                ts->next_tick = 0;
        }

        update_process_times(user_mode(regs));
        profile_tick(CPU_PROFILING);
}

/*
 * We rearm the timer until we get disabled by the idle code.
 * Called with interrupts disabled.
 */
static enum hrtimer_restart tick_nohz_handler(struct hrtimer *timer)
{
        struct tick_sched *ts =        container_of(timer, struct tick_sched, sched_timer);
        struct pt_regs *regs = get_irq_regs();
        ktime_t now = ktime_get();

        tick_sched_do_timer(ts, now);

        /*
         * Do not call when we are not in IRQ context and have
         * no valid 'regs' pointer
         */
        if (regs)
                tick_sched_handle(ts, regs);
        else
                ts->next_tick = 0;

        /*
         * In dynticks mode, tick reprogram is deferred:
         * - to the idle task if in dynticks-idle
         * - to IRQ exit if in full-dynticks.
         */
        if (unlikely(tick_sched_flag_test(ts, TS_FLAG_STOPPED)))
                return HRTIMER_NORESTART;

        hrtimer_forward(timer, now, TICK_NSEC);

        return HRTIMER_RESTART;
}

#ifdef CONFIG_NO_HZ_FULL
cpumask_var_t tick_nohz_full_mask;
EXPORT_SYMBOL_GPL(tick_nohz_full_mask);
bool tick_nohz_full_running;
EXPORT_SYMBOL_GPL(tick_nohz_full_running);
static atomic_t tick_dep_mask;

static bool check_tick_dependency(atomic_t *dep)
{
        int val = atomic_read(dep);

        if (val & TICK_DEP_MASK_POSIX_TIMER) {
                trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
                return true;
        }

        if (val & TICK_DEP_MASK_PERF_EVENTS) {
                trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
                return true;
        }

        if (val & TICK_DEP_MASK_SCHED) {
                trace_tick_stop(0, TICK_DEP_MASK_SCHED);
                return true;
        }

        if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
                trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
                return true;
        }

        if (val & TICK_DEP_MASK_RCU) {
                trace_tick_stop(0, TICK_DEP_MASK_RCU);
                return true;
        }

        if (val & TICK_DEP_MASK_RCU_EXP) {
                trace_tick_stop(0, TICK_DEP_MASK_RCU_EXP);
                return true;
        }

        return false;
}

static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
{
        lockdep_assert_irqs_disabled();

        if (unlikely(!cpu_online(cpu)))
                return false;

        if (check_tick_dependency(&tick_dep_mask))
                return false;

        if (check_tick_dependency(&ts->tick_dep_mask))
                return false;

        if (check_tick_dependency(&current->tick_dep_mask))
                return false;

        if (check_tick_dependency(&current->signal->tick_dep_mask))
                return false;

        return true;
}

static void nohz_full_kick_func(struct irq_work *work)
{
        /* Empty, the tick restart happens on tick_nohz_irq_exit() */
}

static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) =
        IRQ_WORK_INIT_HARD(nohz_full_kick_func);

/*
 * Kick this CPU if it's full dynticks in order to force it to
 * re-evaluate its dependency on the tick and restart it if necessary.
 * This kick, unlike tick_nohz_full_kick_cpu() and tick_nohz_full_kick_all(),
 * is NMI safe.
 */
static void tick_nohz_full_kick(void)
{
        if (!tick_nohz_full_cpu(smp_processor_id()))
                return;

        irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
}

/*
 * Kick the CPU if it's full dynticks in order to force it to
 * re-evaluate its dependency on the tick and restart it if necessary.
 */
void tick_nohz_full_kick_cpu(int cpu)
{
        if (!tick_nohz_full_cpu(cpu))
                return;

        irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
}

static void tick_nohz_kick_task(struct task_struct *tsk)
{
        int cpu;

        /*
         * If the task is not running, run_posix_cpu_timers()
         * has nothing to elapse, and an IPI can then be optimized out.
         *
         * activate_task()                      STORE p->tick_dep_mask
         *   STORE p->on_rq
         * __schedule() (switch to task 'p')    smp_mb() (atomic_fetch_or())
         *   LOCK rq->lock                      LOAD p->on_rq
         *   smp_mb__after_spin_lock()
         *   tick_nohz_task_switch()
         *     LOAD p->tick_dep_mask
         *
         * XXX given a task picks up the dependency on schedule(), should we
         * only care about tasks that are currently on the CPU instead of all
         * that are on the runqueue?
         *
         * That is, does this want to be: task_on_cpu() / task_curr()?
         */
        if (!sched_task_on_rq(tsk))
                return;

        /*
         * If the task concurrently migrates to another CPU,
         * we guarantee it sees the new tick dependency upon
         * schedule.
         *
         * set_task_cpu(p, cpu);
         *   STORE p->cpu = @cpu
         * __schedule() (switch to task 'p')
         *   LOCK rq->lock
         *   smp_mb__after_spin_lock()          STORE p->tick_dep_mask
         *   tick_nohz_task_switch()            smp_mb() (atomic_fetch_or())
         *      LOAD p->tick_dep_mask           LOAD p->cpu
         */
        cpu = task_cpu(tsk);

        preempt_disable();
        if (cpu_online(cpu))
                tick_nohz_full_kick_cpu(cpu);
        preempt_enable();
}

/*
 * Kick all full dynticks CPUs in order to force these to re-evaluate
 * their dependency on the tick and restart it if necessary.
 */
static void tick_nohz_full_kick_all(void)
{
        int cpu;

        if (!tick_nohz_full_running)
                return;

        preempt_disable();
        for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
                tick_nohz_full_kick_cpu(cpu);
        preempt_enable();
}

static void tick_nohz_dep_set_all(atomic_t *dep,
                                  enum tick_dep_bits bit)
{
        int prev;

        prev = atomic_fetch_or(BIT(bit), dep);
        if (!prev)
                tick_nohz_full_kick_all();
}

/*
 * Set a global tick dependency. Used by perf events that rely on freq and
 * unstable clocks.
 */
void tick_nohz_dep_set(enum tick_dep_bits bit)
{
        tick_nohz_dep_set_all(&tick_dep_mask, bit);
}

void tick_nohz_dep_clear(enum tick_dep_bits bit)
{
        atomic_andnot(BIT(bit), &tick_dep_mask);
}

/*
 * Set per-CPU tick dependency. Used by scheduler and perf events in order to
 * manage event-throttling.
 */
void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
{
        int prev;
        struct tick_sched *ts;

        ts = per_cpu_ptr(&tick_cpu_sched, cpu);

        prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
        if (!prev) {
                preempt_disable();
                /* Perf needs local kick that is NMI safe */
                if (cpu == smp_processor_id()) {
                        tick_nohz_full_kick();
                } else {
                        /* Remote IRQ work not NMI-safe */
                        if (!WARN_ON_ONCE(in_nmi()))
                                tick_nohz_full_kick_cpu(cpu);
                }
                preempt_enable();
        }
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);

void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
{
        struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);

        atomic_andnot(BIT(bit), &ts->tick_dep_mask);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);

/*
 * Set a per-task tick dependency. RCU needs this. Also posix CPU timers
 * in order to elapse per task timers.
 */
void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
{
        if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask))
                tick_nohz_kick_task(tsk);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);

void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
{
        atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
}
EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);

/*
 * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse
 * per process timers.
 */
void tick_nohz_dep_set_signal(struct task_struct *tsk,
                              enum tick_dep_bits bit)
{
        int prev;
        struct signal_struct *sig = tsk->signal;

        prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask);
        if (!prev) {
                struct task_struct *t;

                lockdep_assert_held(&tsk->sighand->siglock);
                __for_each_thread(sig, t)
                        tick_nohz_kick_task(t);
        }
}

void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
{
        atomic_andnot(BIT(bit), &sig->tick_dep_mask);
}

/*
 * Re-evaluate the need for the tick as we switch the current task.
 * It might need the tick due to per task/process properties:
 * perf events, posix CPU timers, ...
 */
void __tick_nohz_task_switch(void)
{
        struct tick_sched *ts;

        if (!tick_nohz_full_cpu(smp_processor_id()))
                return;

        ts = this_cpu_ptr(&tick_cpu_sched);

        if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                if (atomic_read(&current->tick_dep_mask) ||
                    atomic_read(&current->signal->tick_dep_mask))
                        tick_nohz_full_kick();
        }
}

/* Get the boot-time nohz CPU list from the kernel parameters. */
void __init tick_nohz_full_setup(cpumask_var_t cpumask)
{
        alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
        cpumask_copy(tick_nohz_full_mask, cpumask);
        tick_nohz_full_running = true;
}

bool tick_nohz_cpu_hotpluggable(unsigned int cpu)
{
        /*
         * The 'tick_do_timer_cpu' CPU handles housekeeping duty (unbound
         * timers, workqueues, timekeeping, ...) on behalf of full dynticks
         * CPUs. It must remain online when nohz full is enabled.
         */
        if (tick_nohz_full_running && READ_ONCE(tick_do_timer_cpu) == cpu)
                return false;
        return true;
}

static int tick_nohz_cpu_down(unsigned int cpu)
{
        return tick_nohz_cpu_hotpluggable(cpu) ? 0 : -EBUSY;
}

void __init tick_nohz_init(void)
{
        int cpu, ret;

        if (!tick_nohz_full_running)
                return;

        /*
         * Full dynticks uses IRQ work to drive the tick rescheduling on safe
         * locking contexts. But then we need IRQ work to raise its own
         * interrupts to avoid circular dependency on the tick.
         */
        if (!arch_irq_work_has_interrupt()) {
                pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support IRQ work self-IPIs\n");
                cpumask_clear(tick_nohz_full_mask);
                tick_nohz_full_running = false;
                return;
        }

        if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
                        !IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
                cpu = smp_processor_id();

                if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
                        pr_warn("NO_HZ: Clearing %d from nohz_full range "
                                "for timekeeping\n", cpu);
                        cpumask_clear_cpu(cpu, tick_nohz_full_mask);
                }
        }

        for_each_cpu(cpu, tick_nohz_full_mask)
                ct_cpu_track_user(cpu);

        ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
                                        "kernel/nohz:predown", NULL,
                                        tick_nohz_cpu_down);
        WARN_ON(ret < 0);
        pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
                cpumask_pr_args(tick_nohz_full_mask));
}
#endif /* #ifdef CONFIG_NO_HZ_FULL */

/*
 * NOHZ - aka dynamic tick functionality
 */
#ifdef CONFIG_NO_HZ_COMMON
/*
 * NO HZ enabled ?
 */
bool tick_nohz_enabled __read_mostly  = true;
unsigned long tick_nohz_active  __read_mostly;
/*
 * Enable / Disable tickless mode
 */
static int __init setup_tick_nohz(char *str)
{
        return (kstrtobool(str, &tick_nohz_enabled) == 0);
}

__setup("nohz=", setup_tick_nohz);

bool tick_nohz_tick_stopped(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
}

bool tick_nohz_tick_stopped_cpu(int cpu)
{
        struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);

        return tick_sched_flag_test(ts, TS_FLAG_STOPPED);
}

/**
 * tick_nohz_update_jiffies - update jiffies when idle was interrupted
 * @now: current ktime_t
 *
 * Called from interrupt entry when the CPU was idle
 *
 * In case the sched_tick was stopped on this CPU, we have to check if jiffies
 * must be updated. Otherwise an interrupt handler could use a stale jiffy
 * value. We do this unconditionally on any CPU, as we don't know whether the
 * CPU, which has the update task assigned, is in a long sleep.
 */
static void tick_nohz_update_jiffies(ktime_t now)
{
        unsigned long flags;

        __this_cpu_write(tick_cpu_sched.idle_waketime, now);

        local_irq_save(flags);
        tick_do_update_jiffies64(now);
        local_irq_restore(flags);

        touch_softlockup_watchdog_sched();
}

static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
{
        ktime_t delta;

        if (WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE)))
                return;

        delta = ktime_sub(now, ts->idle_entrytime);

        write_seqcount_begin(&ts->idle_sleeptime_seq);
        if (nr_iowait_cpu(smp_processor_id()) > 0)
                ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
        else
                ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);

        ts->idle_entrytime = now;
        tick_sched_flag_clear(ts, TS_FLAG_IDLE_ACTIVE);
        write_seqcount_end(&ts->idle_sleeptime_seq);

        sched_clock_idle_wakeup_event();
}

static void tick_nohz_start_idle(struct tick_sched *ts)
{
        write_seqcount_begin(&ts->idle_sleeptime_seq);
        ts->idle_entrytime = ktime_get();
        tick_sched_flag_set(ts, TS_FLAG_IDLE_ACTIVE);
        write_seqcount_end(&ts->idle_sleeptime_seq);

        sched_clock_idle_sleep_event();
}

static u64 get_cpu_sleep_time_us(struct tick_sched *ts, ktime_t *sleeptime,
                                 bool compute_delta, u64 *last_update_time)
{
        ktime_t now, idle;
        unsigned int seq;

        if (!tick_nohz_active)
                return -1;

        now = ktime_get();
        if (last_update_time)
                *last_update_time = ktime_to_us(now);

        do {
                seq = read_seqcount_begin(&ts->idle_sleeptime_seq);

                if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE) && compute_delta) {
                        ktime_t delta = ktime_sub(now, ts->idle_entrytime);

                        idle = ktime_add(*sleeptime, delta);
                } else {
                        idle = *sleeptime;
                }
        } while (read_seqcount_retry(&ts->idle_sleeptime_seq, seq));

        return ktime_to_us(idle);

}

/**
 * get_cpu_idle_time_us - get the total idle time of a CPU
 * @cpu: CPU number to query
 * @last_update_time: variable to store update time in. Do not update
 * counters if NULL.
 *
 * Return the cumulative idle time (since boot) for a given
 * CPU, in microseconds. Note that this is partially broken due to
 * the counter of iowait tasks that can be remotely updated without
 * any synchronization. Therefore it is possible to observe backward
 * values within two consecutive reads.
 *
 * This time is measured via accounting rather than sampling,
 * and is as accurate as ktime_get() is.
 *
 * Return: -1 if NOHZ is not enabled, else total idle time of the @cpu
 */
u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
{
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);

        return get_cpu_sleep_time_us(ts, &ts->idle_sleeptime,
                                     !nr_iowait_cpu(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);

/**
 * get_cpu_iowait_time_us - get the total iowait time of a CPU
 * @cpu: CPU number to query
 * @last_update_time: variable to store update time in. Do not update
 * counters if NULL.
 *
 * Return the cumulative iowait time (since boot) for a given
 * CPU, in microseconds. Note this is partially broken due to
 * the counter of iowait tasks that can be remotely updated without
 * any synchronization. Therefore it is possible to observe backward
 * values within two consecutive reads.
 *
 * This time is measured via accounting rather than sampling,
 * and is as accurate as ktime_get() is.
 *
 * Return: -1 if NOHZ is not enabled, else total iowait time of @cpu
 */
u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
{
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);

        return get_cpu_sleep_time_us(ts, &ts->iowait_sleeptime,
                                     nr_iowait_cpu(cpu), last_update_time);
}
EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);

static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
{
        hrtimer_cancel(&ts->sched_timer);
        hrtimer_set_expires(&ts->sched_timer, ts->last_tick);

        /* Forward the time to expire in the future */
        hrtimer_forward(&ts->sched_timer, now, TICK_NSEC);

        if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
                hrtimer_start_expires(&ts->sched_timer,
                                      HRTIMER_MODE_ABS_PINNED_HARD);
        } else {
                tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
        }

        /*
         * Reset to make sure the next tick stop doesn't get fooled by past
         * cached clock deadline.
         */
        ts->next_tick = 0;
}

static inline bool local_timer_softirq_pending(void)
{
        return local_timers_pending() & BIT(TIMER_SOFTIRQ);
}

/*
 * Read jiffies and the time when jiffies were updated last
 */
u64 get_jiffies_update(unsigned long *basej)
{
        unsigned long basejiff;
        unsigned int seq;
        u64 basemono;

        do {
                seq = read_seqcount_begin(&jiffies_seq);
                basemono = last_jiffies_update;
                basejiff = jiffies;
        } while (read_seqcount_retry(&jiffies_seq, seq));
        *basej = basejiff;
        return basemono;
}

/**
 * tick_nohz_next_event() - return the clock monotonic based next event
 * @ts:                pointer to tick_sched struct
 * @cpu:        CPU number
 *
 * Return:
 * *%0                - When the next event is a maximum of TICK_NSEC in the future
 *                  and the tick is not stopped yet
 * *%next_event        - Next event based on clock monotonic
 */
static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
{
        u64 basemono, next_tick, delta, expires;
        unsigned long basejiff;
        int tick_cpu;

        basemono = get_jiffies_update(&basejiff);
        ts->last_jiffies = basejiff;
        ts->timer_expires_base = basemono;

        /*
         * Keep the periodic tick, when RCU, architecture or irq_work
         * requests it.
         * Aside of that, check whether the local timer softirq is
         * pending. If so, its a bad idea to call get_next_timer_interrupt(),
         * because there is an already expired timer, so it will request
         * immediate expiry, which rearms the hardware timer with a
         * minimal delta, which brings us back to this place
         * immediately. Lather, rinse and repeat...
         */
        if (rcu_needs_cpu() || arch_needs_cpu() ||
            irq_work_needs_cpu() || local_timer_softirq_pending()) {
                next_tick = basemono + TICK_NSEC;
        } else {
                /*
                 * Get the next pending timer. If high resolution
                 * timers are enabled this only takes the timer wheel
                 * timers into account. If high resolution timers are
                 * disabled this also looks at the next expiring
                 * hrtimer.
                 */
                next_tick = get_next_timer_interrupt(basejiff, basemono);
                ts->next_timer = next_tick;
        }

        /* Make sure next_tick is never before basemono! */
        if (WARN_ON_ONCE(basemono > next_tick))
                next_tick = basemono;

        /*
         * If the tick is due in the next period, keep it ticking or
         * force prod the timer.
         */
        delta = next_tick - basemono;
        if (delta <= (u64)TICK_NSEC) {
                /*
                 * We've not stopped the tick yet, and there's a timer in the
                 * next period, so no point in stopping it either, bail.
                 */
                if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                        ts->timer_expires = 0;
                        goto out;
                }
        }

        /*
         * If this CPU is the one which had the do_timer() duty last, we limit
         * the sleep time to the timekeeping 'max_deferment' value.
         * Otherwise we can sleep as long as we want.
         */
        delta = timekeeping_max_deferment();
        tick_cpu = READ_ONCE(tick_do_timer_cpu);
        if (tick_cpu != cpu &&
            (tick_cpu != TICK_DO_TIMER_NONE || !tick_sched_flag_test(ts, TS_FLAG_DO_TIMER_LAST)))
                delta = KTIME_MAX;

        /* Calculate the next expiry time */
        if (delta < (KTIME_MAX - basemono))
                expires = basemono + delta;
        else
                expires = KTIME_MAX;

        ts->timer_expires = min_t(u64, expires, next_tick);

out:
        return ts->timer_expires;
}

static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
{
        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
        unsigned long basejiff = ts->last_jiffies;
        u64 basemono = ts->timer_expires_base;
        bool timer_idle = tick_sched_flag_test(ts, TS_FLAG_STOPPED);
        int tick_cpu;
        u64 expires;

        /* Make sure we won't be trying to stop it twice in a row. */
        ts->timer_expires_base = 0;

        /*
         * Now the tick should be stopped definitely - so the timer base needs
         * to be marked idle as well to not miss a newly queued timer.
         */
        expires = timer_base_try_to_set_idle(basejiff, basemono, &timer_idle);
        if (expires > ts->timer_expires) {
                /*
                 * This path could only happen when the first timer was removed
                 * between calculating the possible sleep length and now (when
                 * high resolution mode is not active, timer could also be a
                 * hrtimer).
                 *
                 * We have to stick to the original calculated expiry value to
                 * not stop the tick for too long with a shallow C-state (which
                 * was programmed by cpuidle because of an early next expiration
                 * value).
                 */
                expires = ts->timer_expires;
        }

        /* If the timer base is not idle, retain the not yet stopped tick. */
        if (!timer_idle)
                return;

        /*
         * If this CPU is the one which updates jiffies, then give up
         * the assignment and let it be taken by the CPU which runs
         * the tick timer next, which might be this CPU as well. If we
         * don't drop this here, the jiffies might be stale and
         * do_timer() never gets invoked. Keep track of the fact that it
         * was the one which had the do_timer() duty last.
         */
        tick_cpu = READ_ONCE(tick_do_timer_cpu);
        if (tick_cpu == cpu) {
                WRITE_ONCE(tick_do_timer_cpu, TICK_DO_TIMER_NONE);
                tick_sched_flag_set(ts, TS_FLAG_DO_TIMER_LAST);
        } else if (tick_cpu != TICK_DO_TIMER_NONE) {
                tick_sched_flag_clear(ts, TS_FLAG_DO_TIMER_LAST);
        }

        /* Skip reprogram of event if it's not changed */
        if (tick_sched_flag_test(ts, TS_FLAG_STOPPED) && (expires == ts->next_tick)) {
                /* Sanity check: make sure clockevent is actually programmed */
                if (expires == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
                        return;

                WARN_ONCE(1, "basemono: %llu ts->next_tick: %llu dev->next_event: %llu "
                          "timer->active: %d timer->expires: %llu\n", basemono, ts->next_tick,
                          dev->next_event, hrtimer_active(&ts->sched_timer),
                          hrtimer_get_expires(&ts->sched_timer));
        }

        /*
         * tick_nohz_stop_tick() can be called several times before
         * tick_nohz_restart_sched_tick() is called. This happens when
         * interrupts arrive which do not cause a reschedule. In the first
         * call we save the current tick time, so we can restart the
         * scheduler tick in tick_nohz_restart_sched_tick().
         */
        if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                calc_load_nohz_start();
                quiet_vmstat();

                ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
                tick_sched_flag_set(ts, TS_FLAG_STOPPED);
                trace_tick_stop(1, TICK_DEP_MASK_NONE);
        }

        ts->next_tick = expires;

        /*
         * If the expiration time == KTIME_MAX, then we simply stop
         * the tick timer.
         */
        if (unlikely(expires == KTIME_MAX)) {
                if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
                        hrtimer_cancel(&ts->sched_timer);
                else
                        tick_program_event(KTIME_MAX, 1);
                return;
        }

        if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES)) {
                hrtimer_start(&ts->sched_timer, expires,
                              HRTIMER_MODE_ABS_PINNED_HARD);
        } else {
                hrtimer_set_expires(&ts->sched_timer, expires);
                tick_program_event(expires, 1);
        }
}

static void tick_nohz_retain_tick(struct tick_sched *ts)
{
        ts->timer_expires_base = 0;
}

#ifdef CONFIG_NO_HZ_FULL
static void tick_nohz_full_stop_tick(struct tick_sched *ts, int cpu)
{
        if (tick_nohz_next_event(ts, cpu))
                tick_nohz_stop_tick(ts, cpu);
        else
                tick_nohz_retain_tick(ts);
}
#endif /* CONFIG_NO_HZ_FULL */

static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
{
        /* Update jiffies first */
        tick_do_update_jiffies64(now);

        /*
         * Clear the timer idle flag, so we avoid IPIs on remote queueing and
         * the clock forward checks in the enqueue path:
         */
        timer_clear_idle();

        calc_load_nohz_stop();
        touch_softlockup_watchdog_sched();

        /* Cancel the scheduled timer and restore the tick: */
        tick_sched_flag_clear(ts, TS_FLAG_STOPPED);
        tick_nohz_restart(ts, now);
}

static void __tick_nohz_full_update_tick(struct tick_sched *ts,
                                         ktime_t now)
{
#ifdef CONFIG_NO_HZ_FULL
        int cpu = smp_processor_id();

        if (can_stop_full_tick(cpu, ts))
                tick_nohz_full_stop_tick(ts, cpu);
        else if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
                tick_nohz_restart_sched_tick(ts, now);
#endif
}

static void tick_nohz_full_update_tick(struct tick_sched *ts)
{
        if (!tick_nohz_full_cpu(smp_processor_id()))
                return;

        if (!tick_sched_flag_test(ts, TS_FLAG_NOHZ))
                return;

        __tick_nohz_full_update_tick(ts, ktime_get());
}

/*
 * A pending softirq outside an IRQ (or softirq disabled section) context
 * should be waiting for ksoftirqd to handle it. Therefore we shouldn't
 * reach this code due to the need_resched() early check in can_stop_idle_tick().
 *
 * However if we are between CPUHP_AP_SMPBOOT_THREADS and CPU_TEARDOWN_CPU on the
 * cpu_down() process, softirqs can still be raised while ksoftirqd is parked,
 * triggering the code below, since wakep_softirqd() is ignored.
 *
 */
static bool report_idle_softirq(void)
{
        static int ratelimit;
        unsigned int pending = local_softirq_pending();

        if (likely(!pending))
                return false;

        /* Some softirqs claim to be safe against hotplug and ksoftirqd parking */
        if (!cpu_active(smp_processor_id())) {
                pending &= ~SOFTIRQ_HOTPLUG_SAFE_MASK;
                if (!pending)
                        return false;
        }

        if (ratelimit >= 10)
                return false;

        /* On RT, softirq handling may be waiting on some lock */
        if (local_bh_blocked())
                return false;

        pr_warn("NOHZ tick-stop error: local softirq work is pending, handler #%02x!!!\n",
                pending);
        ratelimit++;

        return true;
}

static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
{
        WARN_ON_ONCE(cpu_is_offline(cpu));

        if (unlikely(!tick_sched_flag_test(ts, TS_FLAG_NOHZ)))
                return false;

        if (need_resched())
                return false;

        if (unlikely(report_idle_softirq()))
                return false;

        if (tick_nohz_full_enabled()) {
                int tick_cpu = READ_ONCE(tick_do_timer_cpu);

                /*
                 * Keep the tick alive to guarantee timekeeping progression
                 * if there are full dynticks CPUs around
                 */
                if (tick_cpu == cpu)
                        return false;

                /* Should not happen for nohz-full */
                if (WARN_ON_ONCE(tick_cpu == TICK_DO_TIMER_NONE))
                        return false;
        }

        return true;
}

/**
 * tick_nohz_idle_stop_tick - stop the idle tick from the idle task
 *
 * When the next event is more than a tick into the future, stop the idle tick
 */
void tick_nohz_idle_stop_tick(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        int cpu = smp_processor_id();
        ktime_t expires;

        /*
         * If tick_nohz_get_sleep_length() ran tick_nohz_next_event(), the
         * tick timer expiration time is known already.
         */
        if (ts->timer_expires_base)
                expires = ts->timer_expires;
        else if (can_stop_idle_tick(cpu, ts))
                expires = tick_nohz_next_event(ts, cpu);
        else
                return;

        ts->idle_calls++;

        if (expires > 0LL) {
                int was_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);

                tick_nohz_stop_tick(ts, cpu);

                ts->idle_sleeps++;
                ts->idle_expires = expires;

                if (!was_stopped && tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                        ts->idle_jiffies = ts->last_jiffies;
                        nohz_balance_enter_idle(cpu);
                }
        } else {
                tick_nohz_retain_tick(ts);
        }
}

void tick_nohz_idle_retain_tick(void)
{
        tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
}

/**
 * tick_nohz_idle_enter - prepare for entering idle on the current CPU
 *
 * Called when we start the idle loop.
 */
void tick_nohz_idle_enter(void)
{
        struct tick_sched *ts;

        lockdep_assert_irqs_enabled();

        local_irq_disable();

        ts = this_cpu_ptr(&tick_cpu_sched);

        WARN_ON_ONCE(ts->timer_expires_base);

        tick_sched_flag_set(ts, TS_FLAG_INIDLE);
        tick_nohz_start_idle(ts);

        local_irq_enable();
}

/**
 * tick_nohz_irq_exit - Notify the tick about IRQ exit
 *
 * A timer may have been added/modified/deleted either by the current IRQ,
 * or by another place using this IRQ as a notification. This IRQ may have
 * also updated the RCU callback list. These events may require a
 * re-evaluation of the next tick. Depending on the context:
 *
 * 1) If the CPU is idle and no resched is pending, just proceed with idle
 *    time accounting. The next tick will be re-evaluated on the next idle
 *    loop iteration.
 *
 * 2) If the CPU is nohz_full:
 *
 *    2.1) If there is any tick dependency, restart the tick if stopped.
 *
 *    2.2) If there is no tick dependency, (re-)evaluate the next tick and
 *         stop/update it accordingly.
 */
void tick_nohz_irq_exit(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        if (tick_sched_flag_test(ts, TS_FLAG_INIDLE))
                tick_nohz_start_idle(ts);
        else
                tick_nohz_full_update_tick(ts);
}

/**
 * tick_nohz_idle_got_tick - Check whether or not the tick handler has run
 *
 * Return: %true if the tick handler has run, otherwise %false
 */
bool tick_nohz_idle_got_tick(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        if (ts->got_idle_tick) {
                ts->got_idle_tick = 0;
                return true;
        }
        return false;
}

/**
 * tick_nohz_get_next_hrtimer - return the next expiration time for the hrtimer
 * or the tick, whichever expires first. Note that, if the tick has been
 * stopped, it returns the next hrtimer.
 *
 * Called from power state control code with interrupts disabled
 *
 * Return: the next expiration time
 */
ktime_t tick_nohz_get_next_hrtimer(void)
{
        return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
}

/**
 * tick_nohz_get_sleep_length - return the expected length of the current sleep
 * @delta_next: duration until the next event if the tick cannot be stopped
 *
 * Called from power state control code with interrupts disabled.
 *
 * The return value of this function and/or the value returned by it through the
 * @delta_next pointer can be negative which must be taken into account by its
 * callers.
 *
 * Return: the expected length of the current sleep
 */
ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
{
        struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        int cpu = smp_processor_id();
        /*
         * The idle entry time is expected to be a sufficient approximation of
         * the current time at this point.
         */
        ktime_t now = ts->idle_entrytime;
        ktime_t next_event;

        WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));

        *delta_next = ktime_sub(dev->next_event, now);

        if (!can_stop_idle_tick(cpu, ts))
                return *delta_next;

        next_event = tick_nohz_next_event(ts, cpu);
        if (!next_event)
                return *delta_next;

        /*
         * If the next highres timer to expire is earlier than 'next_event', the
         * idle governor needs to know that.
         */
        next_event = min_t(u64, next_event,
                           hrtimer_next_event_without(&ts->sched_timer));

        return ktime_sub(next_event, now);
}

/**
 * tick_nohz_get_idle_calls_cpu - return the current idle calls counter value
 * for a particular CPU.
 * @cpu: target CPU number
 *
 * Called from the schedutil frequency scaling governor in scheduler context.
 *
 * Return: the current idle calls counter value for @cpu
 */
unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
{
        struct tick_sched *ts = tick_get_tick_sched(cpu);

        return ts->idle_calls;
}

static void tick_nohz_account_idle_time(struct tick_sched *ts,
                                        ktime_t now)
{
        unsigned long ticks;

        ts->idle_exittime = now;

        if (vtime_accounting_enabled_this_cpu())
                return;
        /*
         * We stopped the tick in idle. update_process_times() would miss the
         * time we slept, as it does only a 1 tick accounting.
         * Enforce that this is accounted to idle !
         */
        ticks = jiffies - ts->idle_jiffies;
        /*
         * We might be one off. Do not randomly account a huge number of ticks!
         */
        if (ticks && ticks < LONG_MAX)
                account_idle_ticks(ticks);
}

void tick_nohz_idle_restart_tick(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        if (tick_sched_flag_test(ts, TS_FLAG_STOPPED)) {
                ktime_t now = ktime_get();
                tick_nohz_restart_sched_tick(ts, now);
                tick_nohz_account_idle_time(ts, now);
        }
}

static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now)
{
        if (tick_nohz_full_cpu(smp_processor_id()))
                __tick_nohz_full_update_tick(ts, now);
        else
                tick_nohz_restart_sched_tick(ts, now);

        tick_nohz_account_idle_time(ts, now);
}

/**
 * tick_nohz_idle_exit - Update the tick upon idle task exit
 *
 * When the idle task exits, update the tick depending on the
 * following situations:
 *
 * 1) If the CPU is not in nohz_full mode (most cases), then
 *    restart the tick.
 *
 * 2) If the CPU is in nohz_full mode (corner case):
 *   2.1) If the tick can be kept stopped (no tick dependencies)
 *        then re-evaluate the next tick and try to keep it stopped
 *        as long as possible.
 *   2.2) If the tick has dependencies, restart the tick.
 *
 */
void tick_nohz_idle_exit(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        bool idle_active, tick_stopped;
        ktime_t now;

        local_irq_disable();

        WARN_ON_ONCE(!tick_sched_flag_test(ts, TS_FLAG_INIDLE));
        WARN_ON_ONCE(ts->timer_expires_base);

        tick_sched_flag_clear(ts, TS_FLAG_INIDLE);
        idle_active = tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE);
        tick_stopped = tick_sched_flag_test(ts, TS_FLAG_STOPPED);

        if (idle_active || tick_stopped)
                now = ktime_get();

        if (idle_active)
                tick_nohz_stop_idle(ts, now);

        if (tick_stopped)
                tick_nohz_idle_update_tick(ts, now);

        local_irq_enable();
}

/*
 * In low-resolution mode, the tick handler must be implemented directly
 * at the clockevent level. hrtimer can't be used instead, because its
 * infrastructure actually relies on the tick itself as a backend in
 * low-resolution mode (see hrtimer_run_queues()).
 */
static void tick_nohz_lowres_handler(struct clock_event_device *dev)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        dev->next_event = KTIME_MAX;

        if (likely(tick_nohz_handler(&ts->sched_timer) == HRTIMER_RESTART))
                tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
}

static inline void tick_nohz_activate(struct tick_sched *ts)
{
        if (!tick_nohz_enabled)
                return;
        tick_sched_flag_set(ts, TS_FLAG_NOHZ);
        /* One update is enough */
        if (!test_and_set_bit(0, &tick_nohz_active))
                timers_update_nohz();
}

/**
 * tick_nohz_switch_to_nohz - switch to NOHZ mode
 */
static void tick_nohz_switch_to_nohz(void)
{
        if (!tick_nohz_enabled)
                return;

        if (tick_switch_to_oneshot(tick_nohz_lowres_handler))
                return;

        /*
         * Recycle the hrtimer in 'ts', so we can share the
         * highres code.
         */
        tick_setup_sched_timer(false);
}

static inline void tick_nohz_irq_enter(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
        ktime_t now;

        if (!tick_sched_flag_test(ts, TS_FLAG_STOPPED | TS_FLAG_IDLE_ACTIVE))
                return;
        now = ktime_get();
        if (tick_sched_flag_test(ts, TS_FLAG_IDLE_ACTIVE))
                tick_nohz_stop_idle(ts, now);
        /*
         * If all CPUs are idle we may need to update a stale jiffies value.
         * Note nohz_full is a special case: a timekeeper is guaranteed to stay
         * alive but it might be busy looping with interrupts disabled in some
         * rare case (typically stop machine). So we must make sure we have a
         * last resort.
         */
        if (tick_sched_flag_test(ts, TS_FLAG_STOPPED))
                tick_nohz_update_jiffies(now);
}

#else

static inline void tick_nohz_switch_to_nohz(void) { }
static inline void tick_nohz_irq_enter(void) { }
static inline void tick_nohz_activate(struct tick_sched *ts) { }

#endif /* CONFIG_NO_HZ_COMMON */

/*
 * Called from irq_enter() to notify about the possible interruption of idle()
 */
void tick_irq_enter(void)
{
        tick_check_oneshot_broadcast_this_cpu();
        tick_nohz_irq_enter();
}

static int sched_skew_tick;

static int __init skew_tick(char *str)
{
        get_option(&str, &sched_skew_tick);

        return 0;
}
early_param("skew_tick", skew_tick);

/**
 * tick_setup_sched_timer - setup the tick emulation timer
 * @hrtimer: whether to use the hrtimer or not
 */
void tick_setup_sched_timer(bool hrtimer)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        /* Emulate tick processing via per-CPU hrtimers: */
        hrtimer_setup(&ts->sched_timer, tick_nohz_handler, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);

        if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer)
                tick_sched_flag_set(ts, TS_FLAG_HIGHRES);

        /* Get the next period (per-CPU) */
        hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());

        /* Offset the tick to avert 'jiffies_lock' contention. */
        if (sched_skew_tick) {
                u64 offset = TICK_NSEC >> 1;
                do_div(offset, num_possible_cpus());
                offset *= smp_processor_id();
                hrtimer_add_expires_ns(&ts->sched_timer, offset);
        }

        hrtimer_forward_now(&ts->sched_timer, TICK_NSEC);
        if (IS_ENABLED(CONFIG_HIGH_RES_TIMERS) && hrtimer)
                hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
        else
                tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
        tick_nohz_activate(ts);
}

/*
 * Shut down the tick and make sure the CPU won't try to retake the timekeeping
 * duty before disabling IRQs in idle for the last time.
 */
void tick_sched_timer_dying(int cpu)
{
        struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
        ktime_t idle_sleeptime, iowait_sleeptime;
        unsigned long idle_calls, idle_sleeps;

        /* This must happen before hrtimers are migrated! */
        if (tick_sched_flag_test(ts, TS_FLAG_HIGHRES))
                hrtimer_cancel(&ts->sched_timer);

        idle_sleeptime = ts->idle_sleeptime;
        iowait_sleeptime = ts->iowait_sleeptime;
        idle_calls = ts->idle_calls;
        idle_sleeps = ts->idle_sleeps;
        memset(ts, 0, sizeof(*ts));
        ts->idle_sleeptime = idle_sleeptime;
        ts->iowait_sleeptime = iowait_sleeptime;
        ts->idle_calls = idle_calls;
        ts->idle_sleeps = idle_sleeps;
}

/*
 * Async notification about clocksource changes
 */
void tick_clock_notify(void)
{
        int cpu;

        for_each_possible_cpu(cpu)
                set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
}

/*
 * Async notification about clock event changes
 */
void tick_oneshot_notify(void)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        set_bit(0, &ts->check_clocks);
}

/*
 * Check if a change happened, which makes oneshot possible.
 *
 * Called cyclically from the hrtimer softirq (driven by the timer
 * softirq). 'allow_nohz' signals that we can switch into low-res NOHZ
 * mode, because high resolution timers are disabled (either compile
 * or runtime). Called with interrupts disabled.
 */
int tick_check_oneshot_change(int allow_nohz)
{
        struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);

        if (!test_and_clear_bit(0, &ts->check_clocks))
                return 0;

        if (tick_sched_flag_test(ts, TS_FLAG_NOHZ))
                return 0;

        if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
                return 0;

        if (!allow_nohz)
                return 1;

        tick_nohz_switch_to_nohz();
        return 0;
}




































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MM_TYPES_H
#define _LINUX_MM_TYPES_H

#include <linux/mm_types_task.h>

#include <linux/auxvec.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/rbtree.h>
#include <linux/maple_tree.h>
#include <linux/rwsem.h>
#include <linux/completion.h>
#include <linux/cpumask.h>
#include <linux/uprobes.h>
#include <linux/rcupdate.h>
#include <linux/page-flags-layout.h>
#include <linux/workqueue.h>
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>

#include <asm/mmu.h>

#ifndef AT_VECTOR_SIZE_ARCH
#define AT_VECTOR_SIZE_ARCH 0
#endif
#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1))


struct address_space;
struct futex_private_hash;
struct mem_cgroup;

/*
 * Each physical page in the system has a struct page associated with
 * it to keep track of whatever it is we are using the page for at the
 * moment. Note that we have no way to track which tasks are using
 * a page, though if it is a pagecache page, rmap structures can tell us
 * who is mapping it.
 *
 * If you allocate the page using alloc_pages(), you can use some of the
 * space in struct page for your own purposes.  The five words in the main
 * union are available, except for bit 0 of the first word which must be
 * kept clear.  Many users use this word to store a pointer to an object
 * which is guaranteed to be aligned.  If you use the same storage as
 * page->mapping, you must restore it to NULL before freeing the page.
 *
 * The mapcount field must not be used for own purposes.
 *
 * If you want to use the refcount field, it must be used in such a way
 * that other CPUs temporarily incrementing and then decrementing the
 * refcount does not cause problems.  On receiving the page from
 * alloc_pages(), the refcount will be positive.
 *
 * If you allocate pages of order > 0, you can use some of the fields
 * in each subpage, but you may need to restore some of their values
 * afterwards.
 *
 * SLUB uses cmpxchg_double() to atomically update its freelist and counters.
 * That requires that freelist & counters in struct slab be adjacent and
 * double-word aligned. Because struct slab currently just reinterprets the
 * bits of struct page, we align all struct pages to double-word boundaries,
 * and ensure that 'freelist' is aligned within struct slab.
 */
#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE
#define _struct_page_alignment        __aligned(2 * sizeof(unsigned long))
#else
#define _struct_page_alignment        __aligned(sizeof(unsigned long))
#endif

struct page {
        unsigned long flags;                /* Atomic flags, some possibly
                                         * updated asynchronously */
        /*
         * Five words (20/40 bytes) are available in this union.
         * WARNING: bit 0 of the first word is used for PageTail(). That
         * means the other users of this union MUST NOT use the bit to
         * avoid collision and false-positive PageTail().
         */
        union {
                struct {        /* Page cache and anonymous pages */
                        /**
                         * @lru: Pageout list, eg. active_list protected by
                         * lruvec->lru_lock.  Sometimes used as a generic list
                         * by the page owner.
                         */
                        union {
                                struct list_head lru;

                                /* Or, for the Unevictable "LRU list" slot */
                                struct {
                                        /* Always even, to negate PageTail */
                                        void *__filler;
                                        /* Count page's or folio's mlocks */
                                        unsigned int mlock_count;
                                };

                                /* Or, free page */
                                struct list_head buddy_list;
                                struct list_head pcp_list;
                                struct {
                                        struct llist_node pcp_llist;
                                        unsigned int order;
                                };
                        };
                        /* See page-flags.h for PAGE_MAPPING_FLAGS */
                        struct address_space *mapping;
                        union {
                                pgoff_t __folio_index;                /* Our offset within mapping. */
                                unsigned long share;        /* share count for fsdax */
                        };
                        /**
                         * @private: Mapping-private opaque data.
                         * Usually used for buffer_heads if PagePrivate.
                         * Used for swp_entry_t if swapcache flag set.
                         * Indicates order in the buddy system if PageBuddy.
                         */
                        unsigned long private;
                };
                struct {        /* page_pool used by netstack */
                        /**
                         * @pp_magic: magic value to avoid recycling non
                         * page_pool allocated pages.
                         */
                        unsigned long pp_magic;
                        struct page_pool *pp;
                        unsigned long _pp_mapping_pad;
                        unsigned long dma_addr;
                        atomic_long_t pp_ref_count;
                };
                struct {        /* Tail pages of compound page */
                        unsigned long compound_head;        /* Bit zero is set */
                };
                struct {        /* ZONE_DEVICE pages */
                        /*
                         * The first word is used for compound_head or folio
                         * pgmap
                         */
                        void *_unused_pgmap_compound_head;
                        void *zone_device_data;
                        /*
                         * ZONE_DEVICE private pages are counted as being
                         * mapped so the next 3 words hold the mapping, index,
                         * and private fields from the source anonymous or
                         * page cache page while the page is migrated to device
                         * private memory.
                         * ZONE_DEVICE MEMORY_DEVICE_FS_DAX pages also
                         * use the mapping, index, and private fields when
                         * pmem backed DAX files are mapped.
                         */
                };

                /** @rcu_head: You can use this to free a page by RCU. */
                struct rcu_head rcu_head;
        };

        union {                /* This union is 4 bytes in size. */
                /*
                 * For head pages of typed folios, the value stored here
                 * allows for determining what this page is used for. The
                 * tail pages of typed folios will not store a type
                 * (page_type == _mapcount == -1).
                 *
                 * See page-flags.h for a list of page types which are currently
                 * stored here.
                 *
                 * Owners of typed folios may reuse the lower 16 bit of the
                 * head page page_type field after setting the page type,
                 * but must reset these 16 bit to -1 before clearing the
                 * page type.
                 */
                unsigned int page_type;

                /*
                 * For pages that are part of non-typed folios for which mappings
                 * are tracked via the RMAP, encodes the number of times this page
                 * is directly referenced by a page table.
                 *
                 * Note that the mapcount is always initialized to -1, so that
                 * transitions both from it and to it can be tracked, using
                 * atomic_inc_and_test() and atomic_add_negative(-1).
                 */
                atomic_t _mapcount;
        };

        /* Usage count. *DO NOT USE DIRECTLY*. See page_ref.h */
        atomic_t _refcount;

#ifdef CONFIG_MEMCG
        unsigned long memcg_data;
#elif defined(CONFIG_SLAB_OBJ_EXT)
        unsigned long _unused_slab_obj_exts;
#endif

        /*
         * On machines where all RAM is mapped into kernel address space,
         * we can simply calculate the virtual address. On machines with
         * highmem some memory is mapped into kernel virtual memory
         * dynamically, so we need a place to store that address.
         * Note that this field could be 16 bits on x86 ... ;)
         *
         * Architectures with slow multiplication can define
         * WANT_PAGE_VIRTUAL in asm/page.h
         */
#if defined(WANT_PAGE_VIRTUAL)
        void *virtual;                        /* Kernel virtual address (NULL if
                                           not kmapped, ie. highmem) */
#endif /* WANT_PAGE_VIRTUAL */

#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
        int _last_cpupid;
#endif

#ifdef CONFIG_KMSAN
        /*
         * KMSAN metadata for this page:
         *  - shadow page: every bit indicates whether the corresponding
         *    bit of the original page is initialized (0) or not (1);
         *  - origin page: every 4 bytes contain an id of the stack trace
         *    where the uninitialized value was created.
         */
        struct page *kmsan_shadow;
        struct page *kmsan_origin;
#endif
} _struct_page_alignment;

/*
 * struct encoded_page - a nonexistent type marking this pointer
 *
 * An 'encoded_page' pointer is a pointer to a regular 'struct page', but
 * with the low bits of the pointer indicating extra context-dependent
 * information. Only used in mmu_gather handling, and this acts as a type
 * system check on that use.
 *
 * We only really have two guaranteed bits in general, although you could
 * play with 'struct page' alignment (see CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
 * for more.
 *
 * Use the supplied helper functions to endcode/decode the pointer and bits.
 */
struct encoded_page;

#define ENCODED_PAGE_BITS                        3ul

/* Perform rmap removal after we have flushed the TLB. */
#define ENCODED_PAGE_BIT_DELAY_RMAP                1ul

/*
 * The next item in an encoded_page array is the "nr_pages" argument, specifying
 * the number of consecutive pages starting from this page, that all belong to
 * the same folio. For example, "nr_pages" corresponds to the number of folio
 * references that must be dropped. If this bit is not set, "nr_pages" is
 * implicitly 1.
 */
#define ENCODED_PAGE_BIT_NR_PAGES_NEXT                2ul

static __always_inline struct encoded_page *encode_page(struct page *page, unsigned long flags)
{
        BUILD_BUG_ON(flags > ENCODED_PAGE_BITS);
        return (struct encoded_page *)(flags | (unsigned long)page);
}

static inline unsigned long encoded_page_flags(struct encoded_page *page)
{
        return ENCODED_PAGE_BITS & (unsigned long)page;
}

static inline struct page *encoded_page_ptr(struct encoded_page *page)
{
        return (struct page *)(~ENCODED_PAGE_BITS & (unsigned long)page);
}

static __always_inline struct encoded_page *encode_nr_pages(unsigned long nr)
{
        VM_WARN_ON_ONCE((nr << 2) >> 2 != nr);
        return (struct encoded_page *)(nr << 2);
}

static __always_inline unsigned long encoded_nr_pages(struct encoded_page *page)
{
        return ((unsigned long)page) >> 2;
}

/*
 * A swap entry has to fit into a "unsigned long", as the entry is hidden
 * in the "index" field of the swapper address space.
 */
typedef struct {
        unsigned long val;
} swp_entry_t;

#if defined(CONFIG_MEMCG) || defined(CONFIG_SLAB_OBJ_EXT)
/* We have some extra room after the refcount in tail pages. */
#define NR_PAGES_IN_LARGE_FOLIO
#endif

/*
 * On 32bit, we can cut the required metadata in half, because:
 * (a) PID_MAX_LIMIT implicitly limits the number of MMs we could ever have,
 *     so we can limit MM IDs to 15 bit (32767).
 * (b) We don't expect folios where even a single complete PTE mapping by
 *     one MM would exceed 15 bits (order-15).
 */
#ifdef CONFIG_64BIT
typedef int mm_id_mapcount_t;
#define MM_ID_MAPCOUNT_MAX                INT_MAX
typedef unsigned int mm_id_t;
#else /* !CONFIG_64BIT */
typedef short mm_id_mapcount_t;
#define MM_ID_MAPCOUNT_MAX                SHRT_MAX
typedef unsigned short mm_id_t;
#endif /* CONFIG_64BIT */

/* We implicitly use the dummy ID for init-mm etc. where we never rmap pages. */
#define MM_ID_DUMMY                        0
#define MM_ID_MIN                        (MM_ID_DUMMY + 1)

/*
 * We leave the highest bit of each MM id unused, so we can store a flag
 * in the highest bit of each folio->_mm_id[].
 */
#define MM_ID_BITS                        ((sizeof(mm_id_t) * BITS_PER_BYTE) - 1)
#define MM_ID_MASK                        ((1U << MM_ID_BITS) - 1)
#define MM_ID_MAX                        MM_ID_MASK

/*
 * In order to use bit_spin_lock(), which requires an unsigned long, we
 * operate on folio->_mm_ids when working on flags.
 */
#define FOLIO_MM_IDS_LOCK_BITNUM        MM_ID_BITS
#define FOLIO_MM_IDS_LOCK_BIT                BIT(FOLIO_MM_IDS_LOCK_BITNUM)
#define FOLIO_MM_IDS_SHARED_BITNUM        (2 * MM_ID_BITS + 1)
#define FOLIO_MM_IDS_SHARED_BIT                BIT(FOLIO_MM_IDS_SHARED_BITNUM)

/**
 * struct folio - Represents a contiguous set of bytes.
 * @flags: Identical to the page flags.
 * @lru: Least Recently Used list; tracks how recently this folio was used.
 * @mlock_count: Number of times this folio has been pinned by mlock().
 * @mapping: The file this page belongs to, or refers to the anon_vma for
 *    anonymous memory.
 * @index: Offset within the file, in units of pages.  For anonymous memory,
 *    this is the index from the beginning of the mmap.
 * @share: number of DAX mappings that reference this folio. See
 *    dax_associate_entry.
 * @private: Filesystem per-folio data (see folio_attach_private()).
 * @swap: Used for swp_entry_t if folio_test_swapcache().
 * @_mapcount: Do not access this member directly.  Use folio_mapcount() to
 *    find out how many times this folio is mapped by userspace.
 * @_refcount: Do not access this member directly.  Use folio_ref_count()
 *    to find how many references there are to this folio.
 * @memcg_data: Memory Control Group data.
 * @pgmap: Metadata for ZONE_DEVICE mappings
 * @virtual: Virtual address in the kernel direct map.
 * @_last_cpupid: IDs of last CPU and last process that accessed the folio.
 * @_entire_mapcount: Do not use directly, call folio_entire_mapcount().
 * @_large_mapcount: Do not use directly, call folio_mapcount().
 * @_nr_pages_mapped: Do not use outside of rmap and debug code.
 * @_pincount: Do not use directly, call folio_maybe_dma_pinned().
 * @_nr_pages: Do not use directly, call folio_nr_pages().
 * @_mm_id: Do not use outside of rmap code.
 * @_mm_ids: Do not use outside of rmap code.
 * @_mm_id_mapcount: Do not use outside of rmap code.
 * @_hugetlb_subpool: Do not use directly, use accessor in hugetlb.h.
 * @_hugetlb_cgroup: Do not use directly, use accessor in hugetlb_cgroup.h.
 * @_hugetlb_cgroup_rsvd: Do not use directly, use accessor in hugetlb_cgroup.h.
 * @_hugetlb_hwpoison: Do not use directly, call raw_hwp_list_head().
 * @_deferred_list: Folios to be split under memory pressure.
 * @_unused_slab_obj_exts: Placeholder to match obj_exts in struct slab.
 *
 * A folio is a physically, virtually and logically contiguous set
 * of bytes.  It is a power-of-two in size, and it is aligned to that
 * same power-of-two.  It is at least as large as %PAGE_SIZE.  If it is
 * in the page cache, it is at a file offset which is a multiple of that
 * power-of-two.  It may be mapped into userspace at an address which is
 * at an arbitrary page offset, but its kernel virtual address is aligned
 * to its size.
 */
struct folio {
        /* private: don't document the anon union */
        union {
                struct {
        /* public: */
                        unsigned long flags;
                        union {
                                struct list_head lru;
        /* private: avoid cluttering the output */
                                struct {
                                        void *__filler;
        /* public: */
                                        unsigned int mlock_count;
        /* private: */
                                };
        /* public: */
                                struct dev_pagemap *pgmap;
                        };
                        struct address_space *mapping;
                        union {
                                pgoff_t index;
                                unsigned long share;
                        };
                        union {
                                void *private;
                                swp_entry_t swap;
                        };
                        atomic_t _mapcount;
                        atomic_t _refcount;
#ifdef CONFIG_MEMCG
                        unsigned long memcg_data;
#elif defined(CONFIG_SLAB_OBJ_EXT)
                        unsigned long _unused_slab_obj_exts;
#endif
#if defined(WANT_PAGE_VIRTUAL)
                        void *virtual;
#endif
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
                        int _last_cpupid;
#endif
        /* private: the union with struct page is transitional */
                };
                struct page page;
        };
        union {
                struct {
                        unsigned long _flags_1;
                        unsigned long _head_1;
                        union {
                                struct {
        /* public: */
                                        atomic_t _large_mapcount;
                                        atomic_t _nr_pages_mapped;
#ifdef CONFIG_64BIT
                                        atomic_t _entire_mapcount;
                                        atomic_t _pincount;
#endif /* CONFIG_64BIT */
                                        mm_id_mapcount_t _mm_id_mapcount[2];
                                        union {
                                                mm_id_t _mm_id[2];
                                                unsigned long _mm_ids;
                                        };
        /* private: the union with struct page is transitional */
                                };
                                unsigned long _usable_1[4];
                        };
                        atomic_t _mapcount_1;
                        atomic_t _refcount_1;
        /* public: */
#ifdef NR_PAGES_IN_LARGE_FOLIO
                        unsigned int _nr_pages;
#endif /* NR_PAGES_IN_LARGE_FOLIO */
        /* private: the union with struct page is transitional */
                };
                struct page __page_1;
        };
        union {
                struct {
                        unsigned long _flags_2;
                        unsigned long _head_2;
        /* public: */
                        struct list_head _deferred_list;
#ifndef CONFIG_64BIT
                        atomic_t _entire_mapcount;
                        atomic_t _pincount;
#endif /* !CONFIG_64BIT */
        /* private: the union with struct page is transitional */
                };
                struct page __page_2;
        };
        union {
                struct {
                        unsigned long _flags_3;
                        unsigned long _head_3;
        /* public: */
                        void *_hugetlb_subpool;
                        void *_hugetlb_cgroup;
                        void *_hugetlb_cgroup_rsvd;
                        void *_hugetlb_hwpoison;
        /* private: the union with struct page is transitional */
                };
                struct page __page_3;
        };
};

#define FOLIO_MATCH(pg, fl)                                                \
        static_assert(offsetof(struct page, pg) == offsetof(struct folio, fl))
FOLIO_MATCH(flags, flags);
FOLIO_MATCH(lru, lru);
FOLIO_MATCH(mapping, mapping);
FOLIO_MATCH(compound_head, lru);
FOLIO_MATCH(__folio_index, index);
FOLIO_MATCH(private, private);
FOLIO_MATCH(_mapcount, _mapcount);
FOLIO_MATCH(_refcount, _refcount);
#ifdef CONFIG_MEMCG
FOLIO_MATCH(memcg_data, memcg_data);
#endif
#if defined(WANT_PAGE_VIRTUAL)
FOLIO_MATCH(virtual, virtual);
#endif
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
FOLIO_MATCH(_last_cpupid, _last_cpupid);
#endif
#undef FOLIO_MATCH
#define FOLIO_MATCH(pg, fl)                                                \
        static_assert(offsetof(struct folio, fl) ==                        \
                        offsetof(struct page, pg) + sizeof(struct page))
FOLIO_MATCH(flags, _flags_1);
FOLIO_MATCH(compound_head, _head_1);
FOLIO_MATCH(_mapcount, _mapcount_1);
FOLIO_MATCH(_refcount, _refcount_1);
#undef FOLIO_MATCH
#define FOLIO_MATCH(pg, fl)                                                \
        static_assert(offsetof(struct folio, fl) ==                        \
                        offsetof(struct page, pg) + 2 * sizeof(struct page))
FOLIO_MATCH(flags, _flags_2);
FOLIO_MATCH(compound_head, _head_2);
#undef FOLIO_MATCH
#define FOLIO_MATCH(pg, fl)                                                \
        static_assert(offsetof(struct folio, fl) ==                        \
                        offsetof(struct page, pg) + 3 * sizeof(struct page))
FOLIO_MATCH(flags, _flags_3);
FOLIO_MATCH(compound_head, _head_3);
#undef FOLIO_MATCH

/**
 * struct ptdesc -    Memory descriptor for page tables.
 * @__page_flags:     Same as page flags. Powerpc only.
 * @pt_rcu_head:      For freeing page table pages.
 * @pt_list:          List of used page tables. Used for s390 gmap shadow pages
 *                    (which are not linked into the user page tables) and x86
 *                    pgds.
 * @_pt_pad_1:        Padding that aliases with page's compound head.
 * @pmd_huge_pte:     Protected by ptdesc->ptl, used for THPs.
 * @__page_mapping:   Aliases with page->mapping. Unused for page tables.
 * @pt_index:         Used for s390 gmap.
 * @pt_mm:            Used for x86 pgds.
 * @pt_frag_refcount: For fragmented page table tracking. Powerpc only.
 * @pt_share_count:   Used for HugeTLB PMD page table share count.
 * @_pt_pad_2:        Padding to ensure proper alignment.
 * @ptl:              Lock for the page table.
 * @__page_type:      Same as page->page_type. Unused for page tables.
 * @__page_refcount:  Same as page refcount.
 * @pt_memcg_data:    Memcg data. Tracked for page tables here.
 *
 * This struct overlays struct page for now. Do not modify without a good
 * understanding of the issues.
 */
struct ptdesc {
        unsigned long __page_flags;

        union {
                struct rcu_head pt_rcu_head;
                struct list_head pt_list;
                struct {
                        unsigned long _pt_pad_1;
                        pgtable_t pmd_huge_pte;
                };
        };
        unsigned long __page_mapping;

        union {
                pgoff_t pt_index;
                struct mm_struct *pt_mm;
                atomic_t pt_frag_refcount;
#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
                atomic_t pt_share_count;
#endif
        };

        union {
                unsigned long _pt_pad_2;
#if ALLOC_SPLIT_PTLOCKS
                spinlock_t *ptl;
#else
                spinlock_t ptl;
#endif
        };
        unsigned int __page_type;
        atomic_t __page_refcount;
#ifdef CONFIG_MEMCG
        unsigned long pt_memcg_data;
#endif
};

#define TABLE_MATCH(pg, pt)                                                \
        static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt))
TABLE_MATCH(flags, __page_flags);
TABLE_MATCH(compound_head, pt_list);
TABLE_MATCH(compound_head, _pt_pad_1);
TABLE_MATCH(mapping, __page_mapping);
TABLE_MATCH(__folio_index, pt_index);
TABLE_MATCH(rcu_head, pt_rcu_head);
TABLE_MATCH(page_type, __page_type);
TABLE_MATCH(_refcount, __page_refcount);
#ifdef CONFIG_MEMCG
TABLE_MATCH(memcg_data, pt_memcg_data);
#endif
#undef TABLE_MATCH
static_assert(sizeof(struct ptdesc) <= sizeof(struct page));

#define ptdesc_page(pt)                        (_Generic((pt),                        \
        const struct ptdesc *:                (const struct page *)(pt),        \
        struct ptdesc *:                (struct page *)(pt)))

#define ptdesc_folio(pt)                (_Generic((pt),                        \
        const struct ptdesc *:                (const struct folio *)(pt),        \
        struct ptdesc *:                (struct folio *)(pt)))

#define page_ptdesc(p)                        (_Generic((p),                        \
        const struct page *:                (const struct ptdesc *)(p),        \
        struct page *:                        (struct ptdesc *)(p)))

#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING
static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc)
{
        atomic_set(&ptdesc->pt_share_count, 0);
}

static inline void ptdesc_pmd_pts_inc(struct ptdesc *ptdesc)
{
        atomic_inc(&ptdesc->pt_share_count);
}

static inline void ptdesc_pmd_pts_dec(struct ptdesc *ptdesc)
{
        atomic_dec(&ptdesc->pt_share_count);
}

static inline int ptdesc_pmd_pts_count(struct ptdesc *ptdesc)
{
        return atomic_read(&ptdesc->pt_share_count);
}
#else
static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc)
{
}
#endif

/*
 * Used for sizing the vmemmap region on some architectures
 */
#define STRUCT_PAGE_MAX_SHIFT        (order_base_2(sizeof(struct page)))

/*
 * page_private can be used on tail pages.  However, PagePrivate is only
 * checked by the VM on the head page.  So page_private on the tail pages
 * should be used for data that's ancillary to the head page (eg attaching
 * buffer heads to tail pages after attaching buffer heads to the head page)
 */
#define page_private(page)                ((page)->private)

static inline void set_page_private(struct page *page, unsigned long private)
{
        page->private = private;
}

static inline void *folio_get_private(struct folio *folio)
{
        return folio->private;
}

typedef unsigned long vm_flags_t;

/*
 * freeptr_t represents a SLUB freelist pointer, which might be encoded
 * and not dereferenceable if CONFIG_SLAB_FREELIST_HARDENED is enabled.
 */
typedef struct { unsigned long v; } freeptr_t;

/*
 * A region containing a mapping of a non-memory backed file under NOMMU
 * conditions.  These are held in a global tree and are pinned by the VMAs that
 * map parts of them.
 */
struct vm_region {
        struct rb_node        vm_rb;                /* link in global region tree */
        vm_flags_t        vm_flags;        /* VMA vm_flags */
        unsigned long        vm_start;        /* start address of region */
        unsigned long        vm_end;                /* region initialised to here */
        unsigned long        vm_top;                /* region allocated to here */
        unsigned long        vm_pgoff;        /* the offset in vm_file corresponding to vm_start */
        struct file        *vm_file;        /* the backing file or NULL */

        int                vm_usage;        /* region usage count (access under nommu_region_sem) */
        bool                vm_icache_flushed : 1; /* true if the icache has been flushed for
                                                * this region */
};

#ifdef CONFIG_USERFAULTFD
#define NULL_VM_UFFD_CTX ((struct vm_userfaultfd_ctx) { NULL, })
struct vm_userfaultfd_ctx {
        struct userfaultfd_ctx *ctx;
};
#else /* CONFIG_USERFAULTFD */
#define NULL_VM_UFFD_CTX ((struct vm_userfaultfd_ctx) {})
struct vm_userfaultfd_ctx {};
#endif /* CONFIG_USERFAULTFD */

struct anon_vma_name {
        struct kref kref;
        /* The name needs to be at the end because it is dynamically sized. */
        char name[];
};

#ifdef CONFIG_ANON_VMA_NAME
/*
 * mmap_lock should be read-locked when calling anon_vma_name(). Caller should
 * either keep holding the lock while using the returned pointer or it should
 * raise anon_vma_name refcount before releasing the lock.
 */
struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma);
struct anon_vma_name *anon_vma_name_alloc(const char *name);
void anon_vma_name_free(struct kref *kref);
#else /* CONFIG_ANON_VMA_NAME */
static inline struct anon_vma_name *anon_vma_name(struct vm_area_struct *vma)
{
        return NULL;
}

static inline struct anon_vma_name *anon_vma_name_alloc(const char *name)
{
        return NULL;
}
#endif

#define VMA_LOCK_OFFSET        0x40000000
#define VMA_REF_LIMIT        (VMA_LOCK_OFFSET - 1)

struct vma_numab_state {
        /*
         * Initialised as time in 'jiffies' after which VMA
         * should be scanned.  Delays first scan of new VMA by at
         * least sysctl_numa_balancing_scan_delay:
         */
        unsigned long next_scan;

        /*
         * Time in jiffies when pids_active[] is reset to
         * detect phase change behaviour:
         */
        unsigned long pids_active_reset;

        /*
         * Approximate tracking of PIDs that trapped a NUMA hinting
         * fault. May produce false positives due to hash collisions.
         *
         *   [0] Previous PID tracking
         *   [1] Current PID tracking
         *
         * Window moves after next_pid_reset has expired approximately
         * every VMA_PID_RESET_PERIOD jiffies:
         */
        unsigned long pids_active[2];

        /* MM scan sequence ID when scan first started after VMA creation */
        int start_scan_seq;

        /*
         * MM scan sequence ID when the VMA was last completely scanned.
         * A VMA is not eligible for scanning if prev_scan_seq == numa_scan_seq
         */
        int prev_scan_seq;
};

#ifdef __HAVE_PFNMAP_TRACKING
struct pfnmap_track_ctx {
        struct kref kref;
        unsigned long pfn;
        unsigned long size;        /* in bytes */
};
#endif

/*
 * Describes a VMA that is about to be mmap()'ed. Drivers may choose to
 * manipulate mutable fields which will cause those fields to be updated in the
 * resultant VMA.
 *
 * Helper functions are not required for manipulating any field.
 */
struct vm_area_desc {
        /* Immutable state. */
        struct mm_struct *mm;
        unsigned long start;
        unsigned long end;

        /* Mutable fields. Populated with initial state. */
        pgoff_t pgoff;
        struct file *file;
        vm_flags_t vm_flags;
        pgprot_t page_prot;

        /* Write-only fields. */
        const struct vm_operations_struct *vm_ops;
        void *private_data;
};

/*
 * This struct describes a virtual memory area. There is one of these
 * per VM-area/task. A VM area is any part of the process virtual memory
 * space that has a special rule for the page-fault handlers (ie a shared
 * library, the executable area etc).
 *
 * Only explicitly marked struct members may be accessed by RCU readers before
 * getting a stable reference.
 *
 * WARNING: when adding new members, please update vm_area_init_from() to copy
 * them during vm_area_struct content duplication.
 */
struct vm_area_struct {
        /* The first cache line has the info for VMA tree walking. */

        union {
                struct {
                        /* VMA covers [vm_start; vm_end) addresses within mm */
                        unsigned long vm_start;
                        unsigned long vm_end;
                };
                freeptr_t vm_freeptr; /* Pointer used by SLAB_TYPESAFE_BY_RCU */
        };

        /*
         * The address space we belong to.
         * Unstable RCU readers are allowed to read this.
         */
        struct mm_struct *vm_mm;
        pgprot_t vm_page_prot;          /* Access permissions of this VMA. */

        /*
         * Flags, see mm.h.
         * To modify use vm_flags_{init|reset|set|clear|mod} functions.
         */
        union {
                const vm_flags_t vm_flags;
                vm_flags_t __private __vm_flags;
        };

#ifdef CONFIG_PER_VMA_LOCK
        /*
         * Can only be written (using WRITE_ONCE()) while holding both:
         *  - mmap_lock (in write mode)
         *  - vm_refcnt bit at VMA_LOCK_OFFSET is set
         * Can be read reliably while holding one of:
         *  - mmap_lock (in read or write mode)
         *  - vm_refcnt bit at VMA_LOCK_OFFSET is set or vm_refcnt > 1
         * Can be read unreliably (using READ_ONCE()) for pessimistic bailout
         * while holding nothing (except RCU to keep the VMA struct allocated).
         *
         * This sequence counter is explicitly allowed to overflow; sequence
         * counter reuse can only lead to occasional unnecessary use of the
         * slowpath.
         */
        unsigned int vm_lock_seq;
#endif
        /*
         * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma
         * list, after a COW of one of the file pages.        A MAP_SHARED vma
         * can only be in the i_mmap tree.  An anonymous MAP_PRIVATE, stack
         * or brk vma (with NULL file) can only be in an anon_vma list.
         */
        struct list_head anon_vma_chain; /* Serialized by mmap_lock &
                                          * page_table_lock */
        struct anon_vma *anon_vma;        /* Serialized by page_table_lock */

        /* Function pointers to deal with this struct. */
        const struct vm_operations_struct *vm_ops;

        /* Information about our backing store: */
        unsigned long vm_pgoff;                /* Offset (within vm_file) in PAGE_SIZE
                                           units */
        struct file * vm_file;                /* File we map to (can be NULL). */
        void * vm_private_data;                /* was vm_pte (shared mem) */

#ifdef CONFIG_SWAP
        atomic_long_t swap_readahead_info;
#endif
#ifndef CONFIG_MMU
        struct vm_region *vm_region;        /* NOMMU mapping region */
#endif
#ifdef CONFIG_NUMA
        struct mempolicy *vm_policy;        /* NUMA policy for the VMA */
#endif
#ifdef CONFIG_NUMA_BALANCING
        struct vma_numab_state *numab_state;        /* NUMA Balancing state */
#endif
#ifdef CONFIG_PER_VMA_LOCK
        /* Unstable RCU readers are allowed to read this. */
        refcount_t vm_refcnt ____cacheline_aligned_in_smp;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map vmlock_dep_map;
#endif
#endif
        /*
         * For areas with an address space and backing store,
         * linkage into the address_space->i_mmap interval tree.
         *
         */
        struct {
                struct rb_node rb;
                unsigned long rb_subtree_last;
        } shared;
#ifdef CONFIG_ANON_VMA_NAME
        /*
         * For private and shared anonymous mappings, a pointer to a null
         * terminated string containing the name given to the vma, or NULL if
         * unnamed. Serialized by mmap_lock. Use anon_vma_name to access.
         */
        struct anon_vma_name *anon_name;
#endif
        struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
#ifdef __HAVE_PFNMAP_TRACKING
        struct pfnmap_track_ctx *pfnmap_track_ctx;
#endif
} __randomize_layout;

#ifdef CONFIG_NUMA
#define vma_policy(vma) ((vma)->vm_policy)
#else
#define vma_policy(vma) NULL
#endif

#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
        u64 time;
        int cid;
        int recent_cid;
};
#endif

struct kioctx_table;
struct iommu_mm_data;
struct mm_struct {
        struct {
                /*
                 * Fields which are often written to are placed in a separate
                 * cache line.
                 */
                struct {
                        /**
                         * @mm_count: The number of references to &struct
                         * mm_struct (@mm_users count as 1).
                         *
                         * Use mmgrab()/mmdrop() to modify. When this drops to
                         * 0, the &struct mm_struct is freed.
                         */
                        atomic_t mm_count;
                } ____cacheline_aligned_in_smp;

                struct maple_tree mm_mt;

                unsigned long mmap_base;        /* base of mmap area */
                unsigned long mmap_legacy_base;        /* base of mmap area in bottom-up allocations */
#ifdef CONFIG_HAVE_ARCH_COMPAT_MMAP_BASES
                /* Base addresses for compatible mmap() */
                unsigned long mmap_compat_base;
                unsigned long mmap_compat_legacy_base;
#endif
                unsigned long task_size;        /* size of task vm space */
                pgd_t * pgd;

#ifdef CONFIG_MEMBARRIER
                /**
                 * @membarrier_state: Flags controlling membarrier behavior.
                 *
                 * This field is close to @pgd to hopefully fit in the same
                 * cache-line, which needs to be touched by switch_mm().
                 */
                atomic_t membarrier_state;
#endif

                /**
                 * @mm_users: The number of users including userspace.
                 *
                 * Use mmget()/mmget_not_zero()/mmput() to modify. When this
                 * drops to 0 (i.e. when the task exits and there are no other
                 * temporary reference holders), we also release a reference on
                 * @mm_count (which may then free the &struct mm_struct if
                 * @mm_count also drops to 0).
                 */
                atomic_t mm_users;

#ifdef CONFIG_SCHED_MM_CID
                /**
                 * @pcpu_cid: Per-cpu current cid.
                 *
                 * Keep track of the currently allocated mm_cid for each cpu.
                 * The per-cpu mm_cid values are serialized by their respective
                 * runqueue locks.
                 */
                struct mm_cid __percpu *pcpu_cid;
                /*
                 * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
                 *
                 * When the next mm_cid scan is due (in jiffies).
                 */
                unsigned long mm_cid_next_scan;
                /**
                 * @nr_cpus_allowed: Number of CPUs allowed for mm.
                 *
                 * Number of CPUs allowed in the union of all mm's
                 * threads allowed CPUs.
                 */
                unsigned int nr_cpus_allowed;
                /**
                 * @max_nr_cid: Maximum number of allowed concurrency
                 *              IDs allocated.
                 *
                 * Track the highest number of allowed concurrency IDs
                 * allocated for the mm.
                 */
                atomic_t max_nr_cid;
                /**
                 * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
                 *
                 * Provide mutual exclusion for mm cpus_allowed and
                 * mm nr_cpus_allowed updates.
                 */
                raw_spinlock_t cpus_allowed_lock;
#endif
#ifdef CONFIG_MMU
                atomic_long_t pgtables_bytes;        /* size of all page tables */
#endif
                int map_count;                        /* number of VMAs */

                spinlock_t page_table_lock; /* Protects page tables and some
                                             * counters
                                             */
                /*
                 * With some kernel config, the current mmap_lock's offset
                 * inside 'mm_struct' is at 0x120, which is very optimal, as
                 * its two hot fields 'count' and 'owner' sit in 2 different
                 * cachelines,  and when mmap_lock is highly contended, both
                 * of the 2 fields will be accessed frequently, current layout
                 * will help to reduce cache bouncing.
                 *
                 * So please be careful with adding new fields before
                 * mmap_lock, which can easily push the 2 fields into one
                 * cacheline.
                 */
                struct rw_semaphore mmap_lock;

                struct list_head mmlist; /* List of maybe swapped mm's.        These
                                          * are globally strung together off
                                          * init_mm.mmlist, and are protected
                                          * by mmlist_lock
                                          */
#ifdef CONFIG_PER_VMA_LOCK
                struct rcuwait vma_writer_wait;
                /*
                 * This field has lock-like semantics, meaning it is sometimes
                 * accessed with ACQUIRE/RELEASE semantics.
                 * Roughly speaking, incrementing the sequence number is
                 * equivalent to releasing locks on VMAs; reading the sequence
                 * number can be part of taking a read lock on a VMA.
                 * Incremented every time mmap_lock is write-locked/unlocked.
                 * Initialized to 0, therefore odd values indicate mmap_lock
                 * is write-locked and even values that it's released.
                 *
                 * Can be modified under write mmap_lock using RELEASE
                 * semantics.
                 * Can be read with no other protection when holding write
                 * mmap_lock.
                 * Can be read with ACQUIRE semantics if not holding write
                 * mmap_lock.
                 */
                seqcount_t mm_lock_seq;
#endif
#ifdef CONFIG_FUTEX_PRIVATE_HASH
                struct mutex                        futex_hash_lock;
                struct futex_private_hash        __rcu *futex_phash;
                struct futex_private_hash        *futex_phash_new;
#endif

                unsigned long hiwater_rss; /* High-watermark of RSS usage */
                unsigned long hiwater_vm;  /* High-water virtual memory usage */

                unsigned long total_vm;           /* Total pages mapped */
                unsigned long locked_vm;   /* Pages that have PG_mlocked set */
                atomic64_t    pinned_vm;   /* Refcount permanently increased */
                unsigned long data_vm;           /* VM_WRITE & ~VM_SHARED & ~VM_STACK */
                unsigned long exec_vm;           /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
                unsigned long stack_vm;           /* VM_STACK */
                unsigned long def_flags;

                /**
                 * @write_protect_seq: Locked when any thread is write
                 * protecting pages mapped by this mm to enforce a later COW,
                 * for instance during page table copying for fork().
                 */
                seqcount_t write_protect_seq;

                spinlock_t arg_lock; /* protect the below fields */

                unsigned long start_code, end_code, start_data, end_data;
                unsigned long start_brk, brk, start_stack;
                unsigned long arg_start, arg_end, env_start, env_end;

                unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */

                struct percpu_counter rss_stat[NR_MM_COUNTERS];

                struct linux_binfmt *binfmt;

                /* Architecture-specific MM context */
                mm_context_t context;

                unsigned long flags; /* Must use atomic bitops to access */

#ifdef CONFIG_AIO
                spinlock_t                        ioctx_lock;
                struct kioctx_table __rcu        *ioctx_table;
#endif
#ifdef CONFIG_MEMCG
                /*
                 * "owner" points to a task that is regarded as the canonical
                 * user/owner of this mm. All of the following must be true in
                 * order for it to be changed:
                 *
                 * current == mm->owner
                 * current->mm != mm
                 * new_owner->mm == mm
                 * new_owner->alloc_lock is held
                 */
                struct task_struct __rcu *owner;
#endif
                struct user_namespace *user_ns;

                /* store ref to file /proc/<pid>/exe symlink points to */
                struct file __rcu *exe_file;
#ifdef CONFIG_MMU_NOTIFIER
                struct mmu_notifier_subscriptions *notifier_subscriptions;
#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !defined(CONFIG_SPLIT_PMD_PTLOCKS)
                pgtable_t pmd_huge_pte; /* protected by page_table_lock */
#endif
#ifdef CONFIG_NUMA_BALANCING
                /*
                 * numa_next_scan is the next time that PTEs will be remapped
                 * PROT_NONE to trigger NUMA hinting faults; such faults gather
                 * statistics and migrate pages to new nodes if necessary.
                 */
                unsigned long numa_next_scan;

                /* Restart point for scanning and remapping PTEs. */
                unsigned long numa_scan_offset;

                /* numa_scan_seq prevents two threads remapping PTEs. */
                int numa_scan_seq;
#endif
                /*
                 * An operation with batched TLB flushing is going on. Anything
                 * that can move process memory needs to flush the TLB when
                 * moving a PROT_NONE mapped page.
                 */
                atomic_t tlb_flush_pending;
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
                /* See flush_tlb_batched_pending() */
                atomic_t tlb_flush_batched;
#endif
                struct uprobes_state uprobes_state;
#ifdef CONFIG_PREEMPT_RT
                struct rcu_head delayed_drop;
#endif
#ifdef CONFIG_HUGETLB_PAGE
                atomic_long_t hugetlb_usage;
#endif
                struct work_struct async_put_work;

#ifdef CONFIG_IOMMU_MM_DATA
                struct iommu_mm_data *iommu_mm;
#endif
#ifdef CONFIG_KSM
                /*
                 * Represent how many pages of this process are involved in KSM
                 * merging (not including ksm_zero_pages).
                 */
                unsigned long ksm_merging_pages;
                /*
                 * Represent how many pages are checked for ksm merging
                 * including merged and not merged.
                 */
                unsigned long ksm_rmap_items;
                /*
                 * Represent how many empty pages are merged with kernel zero
                 * pages when enabling KSM use_zero_pages.
                 */
                atomic_long_t ksm_zero_pages;
#endif /* CONFIG_KSM */
#ifdef CONFIG_LRU_GEN_WALKS_MMU
                struct {
                        /* this mm_struct is on lru_gen_mm_list */
                        struct list_head list;
                        /*
                         * Set when switching to this mm_struct, as a hint of
                         * whether it has been used since the last time per-node
                         * page table walkers cleared the corresponding bits.
                         */
                        unsigned long bitmap;
#ifdef CONFIG_MEMCG
                        /* points to the memcg of "owner" above */
                        struct mem_cgroup *memcg;
#endif
                } lru_gen;
#endif /* CONFIG_LRU_GEN_WALKS_MMU */
#ifdef CONFIG_MM_ID
                mm_id_t mm_id;
#endif /* CONFIG_MM_ID */
        } __randomize_layout;

        /*
         * The mm_cpumask needs to be at the end of mm_struct, because it
         * is dynamically sized based on nr_cpu_ids.
         */
        unsigned long cpu_bitmap[];
};

#define MM_MT_FLAGS        (MT_FLAGS_ALLOC_RANGE | MT_FLAGS_LOCK_EXTERN | \
                         MT_FLAGS_USE_RCU)
extern struct mm_struct init_mm;

/* Pointer magic because the dynamic array size confuses some compilers. */
static inline void mm_init_cpumask(struct mm_struct *mm)
{
        unsigned long cpu_bitmap = (unsigned long)mm;

        cpu_bitmap += offsetof(struct mm_struct, cpu_bitmap);
        cpumask_clear((struct cpumask *)cpu_bitmap);
}

/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */
static inline cpumask_t *mm_cpumask(struct mm_struct *mm)
{
        return (struct cpumask *)&mm->cpu_bitmap;
}

#ifdef CONFIG_LRU_GEN

struct lru_gen_mm_list {
        /* mm_struct list for page table walkers */
        struct list_head fifo;
        /* protects the list above */
        spinlock_t lock;
};

#endif /* CONFIG_LRU_GEN */

#ifdef CONFIG_LRU_GEN_WALKS_MMU

void lru_gen_add_mm(struct mm_struct *mm);
void lru_gen_del_mm(struct mm_struct *mm);
void lru_gen_migrate_mm(struct mm_struct *mm);

static inline void lru_gen_init_mm(struct mm_struct *mm)
{
        INIT_LIST_HEAD(&mm->lru_gen.list);
        mm->lru_gen.bitmap = 0;
#ifdef CONFIG_MEMCG
        mm->lru_gen.memcg = NULL;
#endif
}

static inline void lru_gen_use_mm(struct mm_struct *mm)
{
        /*
         * When the bitmap is set, page reclaim knows this mm_struct has been
         * used since the last time it cleared the bitmap. So it might be worth
         * walking the page tables of this mm_struct to clear the accessed bit.
         */
        WRITE_ONCE(mm->lru_gen.bitmap, -1);
}

#else /* !CONFIG_LRU_GEN_WALKS_MMU */

static inline void lru_gen_add_mm(struct mm_struct *mm)
{
}

static inline void lru_gen_del_mm(struct mm_struct *mm)
{
}

static inline void lru_gen_migrate_mm(struct mm_struct *mm)
{
}

static inline void lru_gen_init_mm(struct mm_struct *mm)
{
}

static inline void lru_gen_use_mm(struct mm_struct *mm)
{
}

#endif /* CONFIG_LRU_GEN_WALKS_MMU */

struct vma_iterator {
        struct ma_state mas;
};

#define VMA_ITERATOR(name, __mm, __addr)                                \
        struct vma_iterator name = {                                        \
                .mas = {                                                \
                        .tree = &(__mm)->mm_mt,                                \
                        .index = __addr,                                \
                        .node = NULL,                                        \
                        .status = ma_start,                                \
                },                                                        \
        }

static inline void vma_iter_init(struct vma_iterator *vmi,
                struct mm_struct *mm, unsigned long addr)
{
        mas_init(&vmi->mas, &mm->mm_mt, addr);
}

#ifdef CONFIG_SCHED_MM_CID

enum mm_cid_state {
        MM_CID_UNSET = -1U,                /* Unset state has lazy_put flag set. */
        MM_CID_LAZY_PUT = (1U << 31),
};

static inline bool mm_cid_is_unset(int cid)
{
        return cid == MM_CID_UNSET;
}

static inline bool mm_cid_is_lazy_put(int cid)
{
        return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
}

static inline bool mm_cid_is_valid(int cid)
{
        return !(cid & MM_CID_LAZY_PUT);
}

static inline int mm_cid_set_lazy_put(int cid)
{
        return cid | MM_CID_LAZY_PUT;
}

static inline int mm_cid_clear_lazy_put(int cid)
{
        return cid & ~MM_CID_LAZY_PUT;
}

/*
 * mm_cpus_allowed: Union of all mm's threads allowed CPUs.
 */
static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
{
        unsigned long bitmap = (unsigned long)mm;

        bitmap += offsetof(struct mm_struct, cpu_bitmap);
        /* Skip cpu_bitmap */
        bitmap += cpumask_size();
        return (struct cpumask *)bitmap;
}

/* Accessor for struct mm_struct's cidmask. */
static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
{
        unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);

        /* Skip mm_cpus_allowed */
        cid_bitmap += cpumask_size();
        return (struct cpumask *)cid_bitmap;
}

static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
        int i;

        for_each_possible_cpu(i) {
                struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);

                pcpu_cid->cid = MM_CID_UNSET;
                pcpu_cid->recent_cid = MM_CID_UNSET;
                pcpu_cid->time = 0;
        }
        mm->nr_cpus_allowed = p->nr_cpus_allowed;
        atomic_set(&mm->max_nr_cid, 0);
        raw_spin_lock_init(&mm->cpus_allowed_lock);
        cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
        cpumask_clear(mm_cidmask(mm));
}

static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
        mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid);
        if (!mm->pcpu_cid)
                return -ENOMEM;
        mm_init_cid(mm, p);
        return 0;
}
#define mm_alloc_cid(...)        alloc_hooks(mm_alloc_cid_noprof(__VA_ARGS__))

static inline void mm_destroy_cid(struct mm_struct *mm)
{
        free_percpu(mm->pcpu_cid);
        mm->pcpu_cid = NULL;
}

static inline unsigned int mm_cid_size(void)
{
        return 2 * cpumask_size();        /* mm_cpus_allowed(), mm_cidmask(). */
}

static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask)
{
        struct cpumask *mm_allowed = mm_cpus_allowed(mm);

        if (!mm)
                return;
        /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
        raw_spin_lock(&mm->cpus_allowed_lock);
        cpumask_or(mm_allowed, mm_allowed, cpumask);
        WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed));
        raw_spin_unlock(&mm->cpus_allowed_lock);
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
static inline int mm_alloc_cid(struct mm_struct *mm, struct task_struct *p) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }

static inline unsigned int mm_cid_size(void)
{
        return 0;
}
static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask) { }
#endif /* CONFIG_SCHED_MM_CID */

struct mmu_gather;
extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
extern void tlb_finish_mmu(struct mmu_gather *tlb);

struct vm_fault;

/**
 * typedef vm_fault_t - Return type for page fault handlers.
 *
 * Page fault handlers return a bitmask of %VM_FAULT values.
 */
typedef __bitwise unsigned int vm_fault_t;

/**
 * enum vm_fault_reason - Page fault handlers return a bitmask of
 * these values to tell the core VM what happened when handling the
 * fault. Used to decide whether a process gets delivered SIGBUS or
 * just gets major/minor fault counters bumped up.
 *
 * @VM_FAULT_OOM:                Out Of Memory
 * @VM_FAULT_SIGBUS:                Bad access
 * @VM_FAULT_MAJOR:                Page read from storage
 * @VM_FAULT_HWPOISON:                Hit poisoned small page
 * @VM_FAULT_HWPOISON_LARGE:        Hit poisoned large page. Index encoded
 *                                in upper bits
 * @VM_FAULT_SIGSEGV:                segmentation fault
 * @VM_FAULT_NOPAGE:                ->fault installed the pte, not return page
 * @VM_FAULT_LOCKED:                ->fault locked the returned page
 * @VM_FAULT_RETRY:                ->fault blocked, must retry
 * @VM_FAULT_FALLBACK:                huge page fault failed, fall back to small
 * @VM_FAULT_DONE_COW:                ->fault has fully handled COW
 * @VM_FAULT_NEEDDSYNC:                ->fault did not modify page tables and needs
 *                                fsync() to complete (for synchronous page faults
 *                                in DAX)
 * @VM_FAULT_COMPLETED:                ->fault completed, meanwhile mmap lock released
 * @VM_FAULT_HINDEX_MASK:        mask HINDEX value
 *
 */
enum vm_fault_reason {
        VM_FAULT_OOM            = (__force vm_fault_t)0x000001,
        VM_FAULT_SIGBUS         = (__force vm_fault_t)0x000002,
        VM_FAULT_MAJOR          = (__force vm_fault_t)0x000004,
        VM_FAULT_HWPOISON       = (__force vm_fault_t)0x000010,
        VM_FAULT_HWPOISON_LARGE = (__force vm_fault_t)0x000020,
        VM_FAULT_SIGSEGV        = (__force vm_fault_t)0x000040,
        VM_FAULT_NOPAGE         = (__force vm_fault_t)0x000100,
        VM_FAULT_LOCKED         = (__force vm_fault_t)0x000200,
        VM_FAULT_RETRY          = (__force vm_fault_t)0x000400,
        VM_FAULT_FALLBACK       = (__force vm_fault_t)0x000800,
        VM_FAULT_DONE_COW       = (__force vm_fault_t)0x001000,
        VM_FAULT_NEEDDSYNC      = (__force vm_fault_t)0x002000,
        VM_FAULT_COMPLETED      = (__force vm_fault_t)0x004000,
        VM_FAULT_HINDEX_MASK    = (__force vm_fault_t)0x0f0000,
};

/* Encode hstate index for a hwpoisoned large page */
#define VM_FAULT_SET_HINDEX(x) ((__force vm_fault_t)((x) << 16))
#define VM_FAULT_GET_HINDEX(x) (((__force unsigned int)(x) >> 16) & 0xf)

#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS |        \
                        VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON |        \
                        VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK)

#define VM_FAULT_RESULT_TRACE \
        { VM_FAULT_OOM,                 "OOM" },        \
        { VM_FAULT_SIGBUS,              "SIGBUS" },        \
        { VM_FAULT_MAJOR,               "MAJOR" },        \
        { VM_FAULT_HWPOISON,            "HWPOISON" },        \
        { VM_FAULT_HWPOISON_LARGE,      "HWPOISON_LARGE" },        \
        { VM_FAULT_SIGSEGV,             "SIGSEGV" },        \
        { VM_FAULT_NOPAGE,              "NOPAGE" },        \
        { VM_FAULT_LOCKED,              "LOCKED" },        \
        { VM_FAULT_RETRY,               "RETRY" },        \
        { VM_FAULT_FALLBACK,            "FALLBACK" },        \
        { VM_FAULT_DONE_COW,            "DONE_COW" },        \
        { VM_FAULT_NEEDDSYNC,           "NEEDDSYNC" },        \
        { VM_FAULT_COMPLETED,           "COMPLETED" }

struct vm_special_mapping {
        const char *name;        /* The name, e.g. "[vdso]". */

        /*
         * If .fault is not provided, this points to a
         * NULL-terminated array of pages that back the special mapping.
         *
         * This must not be NULL unless .fault is provided.
         */
        struct page **pages;

        /*
         * If non-NULL, then this is called to resolve page faults
         * on the special mapping.  If used, .pages is not checked.
         */
        vm_fault_t (*fault)(const struct vm_special_mapping *sm,
                                struct vm_area_struct *vma,
                                struct vm_fault *vmf);

        int (*mremap)(const struct vm_special_mapping *sm,
                     struct vm_area_struct *new_vma);

        void (*close)(const struct vm_special_mapping *sm,
                      struct vm_area_struct *vma);
};

enum tlb_flush_reason {
        TLB_FLUSH_ON_TASK_SWITCH,
        TLB_REMOTE_SHOOTDOWN,
        TLB_LOCAL_SHOOTDOWN,
        TLB_LOCAL_MM_SHOOTDOWN,
        TLB_REMOTE_SEND_IPI,
        TLB_REMOTE_WRONG_CPU,
        NR_TLB_FLUSH_REASONS,
};

/**
 * enum fault_flag - Fault flag definitions.
 * @FAULT_FLAG_WRITE: Fault was a write fault.
 * @FAULT_FLAG_MKWRITE: Fault was mkwrite of existing PTE.
 * @FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked.
 * @FAULT_FLAG_RETRY_NOWAIT: Don't drop mmap_lock and wait when retrying.
 * @FAULT_FLAG_KILLABLE: The fault task is in SIGKILL killable region.
 * @FAULT_FLAG_TRIED: The fault has been tried once.
 * @FAULT_FLAG_USER: The fault originated in userspace.
 * @FAULT_FLAG_REMOTE: The fault is not for current task/mm.
 * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch.
 * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals.
 * @FAULT_FLAG_UNSHARE: The fault is an unsharing request to break COW in a
 *                      COW mapping, making sure that an exclusive anon page is
 *                      mapped after the fault.
 * @FAULT_FLAG_ORIG_PTE_VALID: whether the fault has vmf->orig_pte cached.
 *                        We should only access orig_pte if this flag set.
 * @FAULT_FLAG_VMA_LOCK: The fault is handled under VMA lock.
 *
 * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify
 * whether we would allow page faults to retry by specifying these two
 * fault flags correctly.  Currently there can be three legal combinations:
 *
 * (a) ALLOW_RETRY and !TRIED:  this means the page fault allows retry, and
 *                              this is the first try
 *
 * (b) ALLOW_RETRY and TRIED:   this means the page fault allows retry, and
 *                              we've already tried at least once
 *
 * (c) !ALLOW_RETRY and !TRIED: this means the page fault does not allow retry
 *
 * The unlisted combination (!ALLOW_RETRY && TRIED) is illegal and should never
 * be used.  Note that page faults can be allowed to retry for multiple times,
 * in which case we'll have an initial fault with flags (a) then later on
 * continuous faults with flags (b).  We should always try to detect pending
 * signals before a retry to make sure the continuous page faults can still be
 * interrupted if necessary.
 *
 * The combination FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE is illegal.
 * FAULT_FLAG_UNSHARE is ignored and treated like an ordinary read fault when
 * applied to mappings that are not COW mappings.
 */
enum fault_flag {
        FAULT_FLAG_WRITE =                1 << 0,
        FAULT_FLAG_MKWRITE =                1 << 1,
        FAULT_FLAG_ALLOW_RETRY =        1 << 2,
        FAULT_FLAG_RETRY_NOWAIT =         1 << 3,
        FAULT_FLAG_KILLABLE =                1 << 4,
        FAULT_FLAG_TRIED =                 1 << 5,
        FAULT_FLAG_USER =                1 << 6,
        FAULT_FLAG_REMOTE =                1 << 7,
        FAULT_FLAG_INSTRUCTION =        1 << 8,
        FAULT_FLAG_INTERRUPTIBLE =        1 << 9,
        FAULT_FLAG_UNSHARE =                1 << 10,
        FAULT_FLAG_ORIG_PTE_VALID =        1 << 11,
        FAULT_FLAG_VMA_LOCK =                1 << 12,
};

typedef unsigned int __bitwise zap_flags_t;

/* Flags for clear_young_dirty_ptes(). */
typedef int __bitwise cydp_t;

/* Clear the access bit */
#define CYDP_CLEAR_YOUNG                ((__force cydp_t)BIT(0))

/* Clear the dirty bit */
#define CYDP_CLEAR_DIRTY                ((__force cydp_t)BIT(1))

/*
 * FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each
 * other. Here is what they mean, and how to use them:
 *
 *
 * FIXME: For pages which are part of a filesystem, mappings are subject to the
 * lifetime enforced by the filesystem and we need guarantees that longterm
 * users like RDMA and V4L2 only establish mappings which coordinate usage with
 * the filesystem.  Ideas for this coordination include revoking the longterm
 * pin, delaying writeback, bounce buffer page writeback, etc.  As FS DAX was
 * added after the problem with filesystems was found FS DAX VMAs are
 * specifically failed.  Filesystem pages are still subject to bugs and use of
 * FOLL_LONGTERM should be avoided on those pages.
 *
 * In the CMA case: long term pins in a CMA region would unnecessarily fragment
 * that region.  And so, CMA attempts to migrate the page before pinning, when
 * FOLL_LONGTERM is specified.
 *
 * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
 * but an additional pin counting system) will be invoked. This is intended for
 * anything that gets a page reference and then touches page data (for example,
 * Direct IO). This lets the filesystem know that some non-file-system entity is
 * potentially changing the pages' data. In contrast to FOLL_GET (whose pages
 * are released via put_page()), FOLL_PIN pages must be released, ultimately, by
 * a call to unpin_user_page().
 *
 * FOLL_PIN is similar to FOLL_GET: both of these pin pages. They use different
 * and separate refcounting mechanisms, however, and that means that each has
 * its own acquire and release mechanisms:
 *
 *     FOLL_GET: get_user_pages*() to acquire, and put_page() to release.
 *
 *     FOLL_PIN: pin_user_pages*() to acquire, and unpin_user_pages to release.
 *
 * FOLL_PIN and FOLL_GET are mutually exclusive for a given function call.
 * (The underlying pages may experience both FOLL_GET-based and FOLL_PIN-based
 * calls applied to them, and that's perfectly OK. This is a constraint on the
 * callers, not on the pages.)
 *
 * FOLL_PIN should be set internally by the pin_user_pages*() APIs, never
 * directly by the caller. That's in order to help avoid mismatches when
 * releasing pages: get_user_pages*() pages must be released via put_page(),
 * while pin_user_pages*() pages must be released via unpin_user_page().
 *
 * Please see Documentation/core-api/pin_user_pages.rst for more information.
 */

enum {
        /* check pte is writable */
        FOLL_WRITE = 1 << 0,
        /* do get_page on page */
        FOLL_GET = 1 << 1,
        /* give error on hole if it would be zero */
        FOLL_DUMP = 1 << 2,
        /* get_user_pages read/write w/o permission */
        FOLL_FORCE = 1 << 3,
        /*
         * if a disk transfer is needed, start the IO and return without waiting
         * upon it
         */
        FOLL_NOWAIT = 1 << 4,
        /* do not fault in pages */
        FOLL_NOFAULT = 1 << 5,
        /* check page is hwpoisoned */
        FOLL_HWPOISON = 1 << 6,
        /* don't do file mappings */
        FOLL_ANON = 1 << 7,
        /*
         * FOLL_LONGTERM indicates that the page will be held for an indefinite
         * time period _often_ under userspace control.  This is in contrast to
         * iov_iter_get_pages(), whose usages are transient.
         */
        FOLL_LONGTERM = 1 << 8,
        /* split huge pmd before returning */
        FOLL_SPLIT_PMD = 1 << 9,
        /* allow returning PCI P2PDMA pages */
        FOLL_PCI_P2PDMA = 1 << 10,
        /* allow interrupts from generic signals */
        FOLL_INTERRUPTIBLE = 1 << 11,
        /*
         * Always honor (trigger) NUMA hinting faults.
         *
         * FOLL_WRITE implicitly honors NUMA hinting faults because a
         * PROT_NONE-mapped page is not writable (exceptions with FOLL_FORCE
         * apply). get_user_pages_fast_only() always implicitly honors NUMA
         * hinting faults.
         */
        FOLL_HONOR_NUMA_FAULT = 1 << 12,

        /* See also internal only FOLL flags in mm/internal.h */
};

/* mm flags */

/*
 * The first two bits represent core dump modes for set-user-ID,
 * the modes are SUID_DUMP_* defined in linux/sched/coredump.h
 */
#define MMF_DUMPABLE_BITS 2
#define MMF_DUMPABLE_MASK ((1 << MMF_DUMPABLE_BITS) - 1)
/* coredump filter bits */
#define MMF_DUMP_ANON_PRIVATE        2
#define MMF_DUMP_ANON_SHARED        3
#define MMF_DUMP_MAPPED_PRIVATE        4
#define MMF_DUMP_MAPPED_SHARED        5
#define MMF_DUMP_ELF_HEADERS        6
#define MMF_DUMP_HUGETLB_PRIVATE 7
#define MMF_DUMP_HUGETLB_SHARED  8
#define MMF_DUMP_DAX_PRIVATE        9
#define MMF_DUMP_DAX_SHARED        10

#define MMF_DUMP_FILTER_SHIFT        MMF_DUMPABLE_BITS
#define MMF_DUMP_FILTER_BITS        9
#define MMF_DUMP_FILTER_MASK \
        (((1 << MMF_DUMP_FILTER_BITS) - 1) << MMF_DUMP_FILTER_SHIFT)
#define MMF_DUMP_FILTER_DEFAULT \
        ((1 << MMF_DUMP_ANON_PRIVATE) |        (1 << MMF_DUMP_ANON_SHARED) |\
         (1 << MMF_DUMP_HUGETLB_PRIVATE) | MMF_DUMP_MASK_DEFAULT_ELF)

#ifdef CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS
# define MMF_DUMP_MASK_DEFAULT_ELF        (1 << MMF_DUMP_ELF_HEADERS)
#else
# define MMF_DUMP_MASK_DEFAULT_ELF        0
#endif
                                        /* leave room for more dump flags */
#define MMF_VM_MERGEABLE        16        /* KSM may merge identical pages */
#define MMF_VM_HUGEPAGE                17        /* set when mm is available for khugepaged */

/*
 * This one-shot flag is dropped due to necessity of changing exe once again
 * on NFS restore
 */
//#define MMF_EXE_FILE_CHANGED        18        /* see prctl_set_mm_exe_file() */

#define MMF_HAS_UPROBES                19        /* has uprobes */
#define MMF_RECALC_UPROBES        20        /* MMF_HAS_UPROBES can be wrong */
#define MMF_OOM_SKIP                21        /* mm is of no interest for the OOM killer */
#define MMF_UNSTABLE                22        /* mm is unstable for copy_from_user */
#define MMF_HUGE_ZERO_PAGE        23      /* mm has ever used the global huge zero page */
#define MMF_DISABLE_THP                24        /* disable THP for all VMAs */
#define MMF_DISABLE_THP_MASK        (1 << MMF_DISABLE_THP)
#define MMF_OOM_REAP_QUEUED        25        /* mm was queued for oom_reaper */
#define MMF_MULTIPROCESS        26        /* mm is shared between processes */
/*
 * MMF_HAS_PINNED: Whether this mm has pinned any pages.  This can be either
 * replaced in the future by mm.pinned_vm when it becomes stable, or grow into
 * a counter on its own. We're aggresive on this bit for now: even if the
 * pinned pages were unpinned later on, we'll still keep this bit set for the
 * lifecycle of this mm, just for simplicity.
 */
#define MMF_HAS_PINNED                27        /* FOLL_PIN has run, never cleared */

#define MMF_HAS_MDWE                28
#define MMF_HAS_MDWE_MASK        (1 << MMF_HAS_MDWE)


#define MMF_HAS_MDWE_NO_INHERIT        29

#define MMF_VM_MERGE_ANY        30
#define MMF_VM_MERGE_ANY_MASK        (1 << MMF_VM_MERGE_ANY)

#define MMF_TOPDOWN                31        /* mm searches top down by default */
#define MMF_TOPDOWN_MASK        (1 << MMF_TOPDOWN)

#define MMF_INIT_MASK                (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\
                                 MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\
                                 MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK)

static inline unsigned long mmf_init_flags(unsigned long flags)
{
        if (flags & (1UL << MMF_HAS_MDWE_NO_INHERIT))
                flags &= ~((1UL << MMF_HAS_MDWE) |
                           (1UL << MMF_HAS_MDWE_NO_INHERIT));
        return flags & MMF_INIT_MASK;
}

#endif /* _LINUX_MM_TYPES_H */















    3 











    1 





    1 











    1 


































































    1 






















    1 




    1 























    1 


































































    2 


    1 


    1 



















    2 


    1 


    1 


















    1 



















































































































































































    1 





    1 








   12 






   11 


    1 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
// SPDX-License-Identifier: GPL-2.0-only
/*
 * VGIC system registers handling functions for AArch64 mode
 */

#include <linux/irqchip/arm-gic-v3.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include "vgic/vgic.h"
#include "sys_regs.h"

static int set_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 val)
{
        u32 host_pri_bits, host_id_bits, host_seis, host_a3v, seis, a3v;
        struct vgic_cpu *vgic_v3_cpu = &vcpu->arch.vgic_cpu;
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);

        /*
         * Disallow restoring VM state if not supported by this
         * hardware.
         */
        host_pri_bits = FIELD_GET(ICC_CTLR_EL1_PRI_BITS_MASK, val) + 1;
        if (host_pri_bits > vgic_v3_cpu->num_pri_bits)
                return -EINVAL;

        vgic_v3_cpu->num_pri_bits = host_pri_bits;

        host_id_bits = FIELD_GET(ICC_CTLR_EL1_ID_BITS_MASK, val);
        if (host_id_bits > vgic_v3_cpu->num_id_bits)
                return -EINVAL;

        vgic_v3_cpu->num_id_bits = host_id_bits;

        host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, kvm_vgic_global_state.ich_vtr_el2);
        seis = FIELD_GET(ICC_CTLR_EL1_SEIS_MASK, val);
        if (host_seis != seis)
                return -EINVAL;

        host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2);
        a3v = FIELD_GET(ICC_CTLR_EL1_A3V_MASK, val);
        if (host_a3v != a3v)
                return -EINVAL;

        /*
         * Here set VMCR.CTLR in ICC_CTLR_EL1 layout.
         * The vgic_set_vmcr() will convert to ICH_VMCR layout.
         */
        vmcr.cbpr = FIELD_GET(ICC_CTLR_EL1_CBPR_MASK, val);
        vmcr.eoim = FIELD_GET(ICC_CTLR_EL1_EOImode_MASK, val);
        vgic_set_vmcr(vcpu, &vmcr);

        return 0;
}

static int get_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 *valp)
{
        struct vgic_cpu *vgic_v3_cpu = &vcpu->arch.vgic_cpu;
        struct vgic_vmcr vmcr;
        u64 val;

        vgic_get_vmcr(vcpu, &vmcr);
        val = 0;
        val |= FIELD_PREP(ICC_CTLR_EL1_PRI_BITS_MASK, vgic_v3_cpu->num_pri_bits - 1);
        val |= FIELD_PREP(ICC_CTLR_EL1_ID_BITS_MASK, vgic_v3_cpu->num_id_bits);
        val |= FIELD_PREP(ICC_CTLR_EL1_SEIS_MASK,
                          FIELD_GET(ICH_VTR_EL2_SEIS,
                                    kvm_vgic_global_state.ich_vtr_el2));
        val |= FIELD_PREP(ICC_CTLR_EL1_A3V_MASK,
                          FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2));
        /*
         * The VMCR.CTLR value is in ICC_CTLR_EL1 layout.
         * Extract it directly using ICC_CTLR_EL1 reg definitions.
         */
        val |= FIELD_PREP(ICC_CTLR_EL1_CBPR_MASK, vmcr.cbpr);
        val |= FIELD_PREP(ICC_CTLR_EL1_EOImode_MASK, vmcr.eoim);

        *valp = val;

        return 0;
}

static int set_gic_pmr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                       u64 val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        vmcr.pmr = FIELD_GET(ICC_PMR_EL1_MASK, val);
        vgic_set_vmcr(vcpu, &vmcr);

        return 0;
}

static int get_gic_pmr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                       u64 *val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        *val = FIELD_PREP(ICC_PMR_EL1_MASK, vmcr.pmr);

        return 0;
}

static int set_gic_bpr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        vmcr.bpr = FIELD_GET(ICC_BPR0_EL1_MASK, val);
        vgic_set_vmcr(vcpu, &vmcr);

        return 0;
}

static int get_gic_bpr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 *val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        *val = FIELD_PREP(ICC_BPR0_EL1_MASK, vmcr.bpr);

        return 0;
}

static int set_gic_bpr1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        if (!vmcr.cbpr) {
                vmcr.abpr = FIELD_GET(ICC_BPR1_EL1_MASK, val);
                vgic_set_vmcr(vcpu, &vmcr);
        }

        return 0;
}

static int get_gic_bpr1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 *val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        if (!vmcr.cbpr)
                *val = FIELD_PREP(ICC_BPR1_EL1_MASK, vmcr.abpr);
        else
                *val = min((vmcr.bpr + 1), 7U);


        return 0;
}

static int set_gic_grpen0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                          u64 val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        vmcr.grpen0 = FIELD_GET(ICC_IGRPEN0_EL1_MASK, val);
        vgic_set_vmcr(vcpu, &vmcr);

        return 0;
}

static int get_gic_grpen0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                          u64 *val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        *val = FIELD_PREP(ICC_IGRPEN0_EL1_MASK, vmcr.grpen0);

        return 0;
}

static int set_gic_grpen1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                          u64 val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        vmcr.grpen1 = FIELD_GET(ICC_IGRPEN1_EL1_MASK, val);
        vgic_set_vmcr(vcpu, &vmcr);

        return 0;
}

static int get_gic_grpen1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                          u64 *val)
{
        struct vgic_vmcr vmcr;

        vgic_get_vmcr(vcpu, &vmcr);
        *val = FIELD_GET(ICC_IGRPEN1_EL1_MASK, vmcr.grpen1);

        return 0;
}

static void set_apr_reg(struct kvm_vcpu *vcpu, u64 val, u8 apr, u8 idx)
{
        struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;

        if (apr)
                vgicv3->vgic_ap1r[idx] = val;
        else
                vgicv3->vgic_ap0r[idx] = val;
}

static u64 get_apr_reg(struct kvm_vcpu *vcpu, u8 apr, u8 idx)
{
        struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;

        if (apr)
                return vgicv3->vgic_ap1r[idx];
        else
                return vgicv3->vgic_ap0r[idx];
}

static int set_gic_ap0r(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 val)

{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        set_apr_reg(vcpu, val, 0, idx);
        return 0;
}

static int get_gic_ap0r(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 *val)
{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        *val = get_apr_reg(vcpu, 0, idx);

        return 0;
}

static int set_gic_ap1r(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 val)

{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        set_apr_reg(vcpu, val, 1, idx);
        return 0;
}

static int get_gic_ap1r(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                        u64 *val)
{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        *val = get_apr_reg(vcpu, 1, idx);

        return 0;
}

static int set_gic_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                       u64 val)
{
        /* Validate SRE bit */
        if (!(val & ICC_SRE_EL1_SRE))
                return -EINVAL;

        return 0;
}

static int get_gic_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                       u64 *val)
{
        struct vgic_v3_cpu_if *vgicv3 = &vcpu->arch.vgic_cpu.vgic_v3;

        *val = vgicv3->vgic_sre;

        return 0;
}

static int set_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 val)
{
        __vcpu_assign_sys_reg(vcpu, r->reg, val);
        return 0;
}

static int get_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                            u64 *val)
{
        *val = __vcpu_sys_reg(vcpu, r->reg);
        return 0;
}

static int set_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 val)
{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        return set_gic_ich_reg(vcpu, r, val);
}

static int get_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                            u64 *val)
{
        u8 idx = r->Op2 & 3;

        if (idx > vgic_v3_max_apr_idx(vcpu))
                return -EINVAL;

        return get_gic_ich_reg(vcpu, r, val);
}

static int set_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 val)
{
        if (val != KVM_ICC_SRE_EL2)
                return -EINVAL;
        return 0;
}

static int get_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 *val)
{
        *val = KVM_ICC_SRE_EL2;
        return 0;
}

static int set_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 val)
{
        if (val != kvm_get_guest_vtr_el2())
                return -EINVAL;
        return 0;
}

static int get_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
                           u64 *val)
{
        *val = kvm_get_guest_vtr_el2();
        return 0;
}

static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
                                   const struct sys_reg_desc *rd)
{
        return vcpu_has_nv(vcpu) ? 0 : REG_HIDDEN;
}

#define __EL2_REG(r, acc, i)                        \
        {                                        \
                SYS_DESC(SYS_ ## r),                \
                .get_user = get_gic_ ## acc,        \
                .set_user = set_gic_ ## acc,        \
                .reg = i,                        \
                .visibility = el2_visibility,        \
        }

#define EL2_REG(r, acc)                __EL2_REG(r, acc, r)

#define EL2_REG_RO(r, acc)        __EL2_REG(r, acc, 0)

static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
        { SYS_DESC(SYS_ICC_PMR_EL1),
          .set_user = set_gic_pmr, .get_user = get_gic_pmr, },
        { SYS_DESC(SYS_ICC_BPR0_EL1),
          .set_user = set_gic_bpr0, .get_user = get_gic_bpr0, },
        { SYS_DESC(SYS_ICC_AP0R0_EL1),
          .set_user = set_gic_ap0r, .get_user = get_gic_ap0r, },
        { SYS_DESC(SYS_ICC_AP0R1_EL1),
          .set_user = set_gic_ap0r, .get_user = get_gic_ap0r, },
        { SYS_DESC(SYS_ICC_AP0R2_EL1),
          .set_user = set_gic_ap0r, .get_user = get_gic_ap0r, },
        { SYS_DESC(SYS_ICC_AP0R3_EL1),
          .set_user = set_gic_ap0r, .get_user = get_gic_ap0r, },
        { SYS_DESC(SYS_ICC_AP1R0_EL1),
          .set_user = set_gic_ap1r, .get_user = get_gic_ap1r, },
        { SYS_DESC(SYS_ICC_AP1R1_EL1),
          .set_user = set_gic_ap1r, .get_user = get_gic_ap1r, },
        { SYS_DESC(SYS_ICC_AP1R2_EL1),
          .set_user = set_gic_ap1r, .get_user = get_gic_ap1r, },
        { SYS_DESC(SYS_ICC_AP1R3_EL1),
          .set_user = set_gic_ap1r, .get_user = get_gic_ap1r, },
        { SYS_DESC(SYS_ICC_BPR1_EL1),
          .set_user = set_gic_bpr1, .get_user = get_gic_bpr1, },
        { SYS_DESC(SYS_ICC_CTLR_EL1),
          .set_user = set_gic_ctlr, .get_user = get_gic_ctlr, },
        { SYS_DESC(SYS_ICC_SRE_EL1),
          .set_user = set_gic_sre, .get_user = get_gic_sre, },
        { SYS_DESC(SYS_ICC_IGRPEN0_EL1),
          .set_user = set_gic_grpen0, .get_user = get_gic_grpen0, },
        { SYS_DESC(SYS_ICC_IGRPEN1_EL1),
          .set_user = set_gic_grpen1, .get_user = get_gic_grpen1, },
        EL2_REG(ICH_AP0R0_EL2, ich_apr),
        EL2_REG(ICH_AP0R1_EL2, ich_apr),
        EL2_REG(ICH_AP0R2_EL2, ich_apr),
        EL2_REG(ICH_AP0R3_EL2, ich_apr),
        EL2_REG(ICH_AP1R0_EL2, ich_apr),
        EL2_REG(ICH_AP1R1_EL2, ich_apr),
        EL2_REG(ICH_AP1R2_EL2, ich_apr),
        EL2_REG(ICH_AP1R3_EL2, ich_apr),
        EL2_REG_RO(ICC_SRE_EL2, icc_sre),
        EL2_REG(ICH_HCR_EL2, ich_reg),
        EL2_REG_RO(ICH_VTR_EL2, ich_vtr),
        EL2_REG(ICH_VMCR_EL2, ich_reg),
        EL2_REG(ICH_LR0_EL2, ich_reg),
        EL2_REG(ICH_LR1_EL2, ich_reg),
        EL2_REG(ICH_LR2_EL2, ich_reg),
        EL2_REG(ICH_LR3_EL2, ich_reg),
        EL2_REG(ICH_LR4_EL2, ich_reg),
        EL2_REG(ICH_LR5_EL2, ich_reg),
        EL2_REG(ICH_LR6_EL2, ich_reg),
        EL2_REG(ICH_LR7_EL2, ich_reg),
        EL2_REG(ICH_LR8_EL2, ich_reg),
        EL2_REG(ICH_LR9_EL2, ich_reg),
        EL2_REG(ICH_LR10_EL2, ich_reg),
        EL2_REG(ICH_LR11_EL2, ich_reg),
        EL2_REG(ICH_LR12_EL2, ich_reg),
        EL2_REG(ICH_LR13_EL2, ich_reg),
        EL2_REG(ICH_LR14_EL2, ich_reg),
        EL2_REG(ICH_LR15_EL2, ich_reg),
};

const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz)
{
        *sz = ARRAY_SIZE(gic_v3_icc_reg_descs);
        return gic_v3_icc_reg_descs;
}

static u64 attr_to_id(u64 attr)
{
        return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr),
                             FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr),
                             FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr),
                             FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr),
                             FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr));
}

int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
        const struct sys_reg_desc *r;

        r = get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
                          ARRAY_SIZE(gic_v3_icc_reg_descs));

        if (r && !sysreg_hidden(vcpu, r))
                return 0;

        return -ENXIO;
}

int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
                                struct kvm_device_attr *attr,
                                bool is_write)
{
        struct kvm_one_reg reg = {
                .id        = attr_to_id(attr->attr),
                .addr        = attr->addr,
        };

        if (is_write)
                return kvm_sys_reg_set_user(vcpu, &reg, gic_v3_icc_reg_descs,
                                            ARRAY_SIZE(gic_v3_icc_reg_descs));
        else
                return kvm_sys_reg_get_user(vcpu, &reg, gic_v3_icc_reg_descs,
                                            ARRAY_SIZE(gic_v3_icc_reg_descs));
}































    3 
   32 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2012,2013 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 */

#ifndef __KVM_ARM_PSCI_H__
#define __KVM_ARM_PSCI_H__

#include <linux/kvm_host.h>
#include <uapi/linux/psci.h>

#define KVM_ARM_PSCI_0_1        PSCI_VERSION(0, 1)
#define KVM_ARM_PSCI_0_2        PSCI_VERSION(0, 2)
#define KVM_ARM_PSCI_1_0        PSCI_VERSION(1, 0)
#define KVM_ARM_PSCI_1_1        PSCI_VERSION(1, 1)
#define KVM_ARM_PSCI_1_2        PSCI_VERSION(1, 2)
#define KVM_ARM_PSCI_1_3        PSCI_VERSION(1, 3)

#define KVM_ARM_PSCI_LATEST        KVM_ARM_PSCI_1_3

static inline int kvm_psci_version(struct kvm_vcpu *vcpu)
{
        /*
         * Our PSCI implementation stays the same across versions from
         * v0.2 onward, only adding the few mandatory functions (such
         * as FEATURES with 1.0) that are required by newer
         * revisions. It is thus safe to return the latest, unless
         * userspace has instructed us otherwise.
         */
        if (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2)) {
                if (vcpu->kvm->arch.psci_version)
                        return vcpu->kvm->arch.psci_version;

                return KVM_ARM_PSCI_LATEST;
        }

        return KVM_ARM_PSCI_0_1;
}


int kvm_psci_call(struct kvm_vcpu *vcpu);

#endif /* __KVM_ARM_PSCI_H__ */





































































































































































































































































































   37 
   37 






   37 





   37 



































































































































  262 
  149 

















  149 








  147 


   86 


  149 







  149 

























  262 





  263 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Based on arch/arm/include/asm/tlbflush.h
 *
 * Copyright (C) 1999-2003 Russell King
 * Copyright (C) 2012 ARM Ltd.
 */
#ifndef __ASM_TLBFLUSH_H
#define __ASM_TLBFLUSH_H

#ifndef __ASSEMBLY__

#include <linux/bitfield.h>
#include <linux/mm_types.h>
#include <linux/sched.h>
#include <linux/mmu_notifier.h>
#include <asm/cputype.h>
#include <asm/mmu.h>

/*
 * Raw TLBI operations.
 *
 * Where necessary, use the __tlbi() macro to avoid asm()
 * boilerplate. Drivers and most kernel code should use the TLB
 * management routines in preference to the macro below.
 *
 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
 * on whether a particular TLBI operation takes an argument or
 * not. The macros handles invoking the asm with or without the
 * register argument as appropriate.
 */
#define __TLBI_0(op, arg) asm (ARM64_ASM_PREAMBLE                               \
                               "tlbi " #op "\n"                                       \
                   ALTERNATIVE("nop\n                        nop",                       \
                               "dsb ish\n                tlbi " #op,               \
                               ARM64_WORKAROUND_REPEAT_TLBI,                       \
                               CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)               \
                            : : )

#define __TLBI_1(op, arg) asm (ARM64_ASM_PREAMBLE                               \
                               "tlbi " #op ", %0\n"                               \
                   ALTERNATIVE("nop\n                        nop",                       \
                               "dsb ish\n                tlbi " #op ", %0",     \
                               ARM64_WORKAROUND_REPEAT_TLBI,                       \
                               CONFIG_ARM64_WORKAROUND_REPEAT_TLBI)               \
                            : : "r" (arg))

#define __TLBI_N(op, arg, n, ...) __TLBI_##n(op, arg)

#define __tlbi(op, ...)                __TLBI_N(op, ##__VA_ARGS__, 1, 0)

#define __tlbi_user(op, arg) do {                                                \
        if (arm64_kernel_unmapped_at_el0())                                        \
                __tlbi(op, (arg) | USER_ASID_FLAG);                                \
} while (0)

/* This macro creates a properly formatted VA operand for the TLBI */
#define __TLBI_VADDR(addr, asid)                                \
        ({                                                        \
                unsigned long __ta = (addr) >> 12;                \
                __ta &= GENMASK_ULL(43, 0);                        \
                __ta |= (unsigned long)(asid) << 48;                \
                __ta;                                                \
        })

/*
 * Get translation granule of the system, which is decided by
 * PAGE_SIZE.  Used by TTL.
 *  - 4KB        : 1
 *  - 16KB        : 2
 *  - 64KB        : 3
 */
#define TLBI_TTL_TG_4K                1
#define TLBI_TTL_TG_16K                2
#define TLBI_TTL_TG_64K                3

static inline unsigned long get_trans_granule(void)
{
        switch (PAGE_SIZE) {
        case SZ_4K:
                return TLBI_TTL_TG_4K;
        case SZ_16K:
                return TLBI_TTL_TG_16K;
        case SZ_64K:
                return TLBI_TTL_TG_64K;
        default:
                return 0;
        }
}

/*
 * Level-based TLBI operations.
 *
 * When ARMv8.4-TTL exists, TLBI operations take an additional hint for
 * the level at which the invalidation must take place. If the level is
 * wrong, no invalidation may take place. In the case where the level
 * cannot be easily determined, the value TLBI_TTL_UNKNOWN will perform
 * a non-hinted invalidation. Any provided level outside the hint range
 * will also cause fall-back to non-hinted invalidation.
 *
 * For Stage-2 invalidation, use the level values provided to that effect
 * in asm/stage2_pgtable.h.
 */
#define TLBI_TTL_MASK                GENMASK_ULL(47, 44)

#define TLBI_TTL_UNKNOWN        INT_MAX

#define __tlbi_level(op, addr, level) do {                                \
        u64 arg = addr;                                                        \
                                                                        \
        if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) &&        \
            level >= 0 && level <= 3) {                                        \
                u64 ttl = level & 3;                                        \
                ttl |= get_trans_granule() << 2;                        \
                arg &= ~TLBI_TTL_MASK;                                        \
                arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);                        \
        }                                                                \
                                                                        \
        __tlbi(op, arg);                                                \
} while(0)

#define __tlbi_user_level(op, arg, level) do {                                \
        if (arm64_kernel_unmapped_at_el0())                                \
                __tlbi_level(op, (arg | USER_ASID_FLAG), level);        \
} while (0)

/*
 * This macro creates a properly formatted VA operand for the TLB RANGE. The
 * value bit assignments are:
 *
 * +----------+------+-------+-------+-------+----------------------+
 * |   ASID   |  TG  | SCALE |  NUM  |  TTL  |        BADDR         |
 * +-----------------+-------+-------+-------+----------------------+
 * |63      48|47  46|45   44|43   39|38   37|36                   0|
 *
 * The address range is determined by below formula: [BADDR, BADDR + (NUM + 1) *
 * 2^(5*SCALE + 1) * PAGESIZE)
 *
 * Note that the first argument, baddr, is pre-shifted; If LPA2 is in use, BADDR
 * holds addr[52:16]. Else BADDR holds page number. See for example ARM DDI
 * 0487J.a section C5.5.60 "TLBI VAE1IS, TLBI VAE1ISNXS, TLB Invalidate by VA,
 * EL1, Inner Shareable".
 *
 */
#define TLBIR_ASID_MASK                GENMASK_ULL(63, 48)
#define TLBIR_TG_MASK                GENMASK_ULL(47, 46)
#define TLBIR_SCALE_MASK        GENMASK_ULL(45, 44)
#define TLBIR_NUM_MASK                GENMASK_ULL(43, 39)
#define TLBIR_TTL_MASK                GENMASK_ULL(38, 37)
#define TLBIR_BADDR_MASK        GENMASK_ULL(36,  0)

#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl)                \
        ({                                                                \
                unsigned long __ta = 0;                                        \
                unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0;        \
                __ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr);                \
                __ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl);                \
                __ta |= FIELD_PREP(TLBIR_NUM_MASK, num);                \
                __ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale);                \
                __ta |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule());        \
                __ta |= FIELD_PREP(TLBIR_ASID_MASK, asid);                \
                __ta;                                                        \
        })

/* These macros are used by the TLBI RANGE feature. */
#define __TLBI_RANGE_PAGES(num, scale)        \
        ((unsigned long)((num) + 1) << (5 * (scale) + 1))
#define MAX_TLBI_RANGE_PAGES                __TLBI_RANGE_PAGES(31, 3)

/*
 * Generate 'num' values from -1 to 31 with -1 rejected by the
 * __flush_tlb_range() loop below. Its return value is only
 * significant for a maximum of MAX_TLBI_RANGE_PAGES pages. If
 * 'pages' is more than that, you must iterate over the overall
 * range.
 */
#define __TLBI_RANGE_NUM(pages, scale)                                        \
        ({                                                                \
                int __pages = min((pages),                                \
                                  __TLBI_RANGE_PAGES(31, (scale)));        \
                (__pages >> (5 * (scale) + 1)) - 1;                        \
        })

/*
 *        TLB Invalidation
 *        ================
 *
 *         This header file implements the low-level TLB invalidation routines
 *        (sometimes referred to as "flushing" in the kernel) for arm64.
 *
 *        Every invalidation operation uses the following template:
 *
 *        DSB ISHST        // Ensure prior page-table updates have completed
 *        TLBI ...        // Invalidate the TLB
 *        DSB ISH                // Ensure the TLB invalidation has completed
 *      if (invalidated kernel mappings)
 *                ISB        // Discard any instructions fetched from the old mapping
 *
 *
 *        The following functions form part of the "core" TLB invalidation API,
 *        as documented in Documentation/core-api/cachetlb.rst:
 *
 *        flush_tlb_all()
 *                Invalidate the entire TLB (kernel + user) on all CPUs
 *
 *        flush_tlb_mm(mm)
 *                Invalidate an entire user address space on all CPUs.
 *                The 'mm' argument identifies the ASID to invalidate.
 *
 *        flush_tlb_range(vma, start, end)
 *                Invalidate the virtual-address range '[start, end)' on all
 *                CPUs for the user address space corresponding to 'vma->mm'.
 *                Note that this operation also invalidates any walk-cache
 *                entries associated with translations for the specified address
 *                range.
 *
 *        flush_tlb_kernel_range(start, end)
 *                Same as flush_tlb_range(..., start, end), but applies to
 *                 kernel mappings rather than a particular user address space.
 *                Whilst not explicitly documented, this function is used when
 *                unmapping pages from vmalloc/io space.
 *
 *        flush_tlb_page(vma, addr)
 *                Invalidate a single user mapping for address 'addr' in the
 *                address space corresponding to 'vma->mm'.  Note that this
 *                operation only invalidates a single, last-level page-table
 *                entry and therefore does not affect any walk-caches.
 *
 *
 *        Next, we have some undocumented invalidation routines that you probably
 *        don't want to call unless you know what you're doing:
 *
 *        local_flush_tlb_all()
 *                Same as flush_tlb_all(), but only applies to the calling CPU.
 *
 *        __flush_tlb_kernel_pgtable(addr)
 *                Invalidate a single kernel mapping for address 'addr' on all
 *                CPUs, ensuring that any walk-cache entries associated with the
 *                translation are also invalidated.
 *
 *        __flush_tlb_range(vma, start, end, stride, last_level, tlb_level)
 *                Invalidate the virtual-address range '[start, end)' on all
 *                CPUs for the user address space corresponding to 'vma->mm'.
 *                The invalidation operations are issued at a granularity
 *                determined by 'stride' and only affect any walk-cache entries
 *                if 'last_level' is equal to false. tlb_level is the level at
 *                which the invalidation must take place. If the level is wrong,
 *                no invalidation may take place. In the case where the level
 *                cannot be easily determined, the value TLBI_TTL_UNKNOWN will
 *                perform a non-hinted invalidation.
 *
 *
 *        Finally, take a look at asm/tlb.h to see how tlb_flush() is implemented
 *        on top of these routines, since that is our interface to the mmu_gather
 *        API as used by munmap() and friends.
 */
static inline void local_flush_tlb_all(void)
{
        dsb(nshst);
        __tlbi(vmalle1);
        dsb(nsh);
        isb();
}

static inline void flush_tlb_all(void)
{
        dsb(ishst);
        __tlbi(vmalle1is);
        dsb(ish);
        isb();
}

static inline void flush_tlb_mm(struct mm_struct *mm)
{
        unsigned long asid;

        dsb(ishst);
        asid = __TLBI_VADDR(0, ASID(mm));
        __tlbi(aside1is, asid);
        __tlbi_user(aside1is, asid);
        dsb(ish);
        mmu_notifier_arch_invalidate_secondary_tlbs(mm, 0, -1UL);
}

static inline void __flush_tlb_page_nosync(struct mm_struct *mm,
                                           unsigned long uaddr)
{
        unsigned long addr;

        dsb(ishst);
        addr = __TLBI_VADDR(uaddr, ASID(mm));
        __tlbi(vale1is, addr);
        __tlbi_user(vale1is, addr);
        mmu_notifier_arch_invalidate_secondary_tlbs(mm, uaddr & PAGE_MASK,
                                                (uaddr & PAGE_MASK) + PAGE_SIZE);
}

static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
                                         unsigned long uaddr)
{
        return __flush_tlb_page_nosync(vma->vm_mm, uaddr);
}

static inline void flush_tlb_page(struct vm_area_struct *vma,
                                  unsigned long uaddr)
{
        flush_tlb_page_nosync(vma, uaddr);
        dsb(ish);
}

static inline bool arch_tlbbatch_should_defer(struct mm_struct *mm)
{
        /*
         * TLB flush deferral is not required on systems which are affected by
         * ARM64_WORKAROUND_REPEAT_TLBI, as __tlbi()/__tlbi_user() implementation
         * will have two consecutive TLBI instructions with a dsb(ish) in between
         * defeating the purpose (i.e save overall 'dsb ish' cost).
         */
        if (alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI))
                return false;

        return true;
}

/*
 * If mprotect/munmap/etc occurs during TLB batched flushing, we need to ensure
 * all the previously issued TLBIs targeting mm have completed. But since we
 * can be executing on a remote CPU, a DSB cannot guarantee this like it can
 * for arch_tlbbatch_flush(). Our only option is to flush the entire mm.
 */
static inline void arch_flush_tlb_batched_pending(struct mm_struct *mm)
{
        flush_tlb_mm(mm);
}

/*
 * To support TLB batched flush for multiple pages unmapping, we only send
 * the TLBI for each page in arch_tlbbatch_add_pending() and wait for the
 * completion at the end in arch_tlbbatch_flush(). Since we've already issued
 * TLBI for each page so only a DSB is needed to synchronise its effect on the
 * other CPUs.
 *
 * This will save the time waiting on DSB comparing issuing a TLBI;DSB sequence
 * for each page.
 */
static inline void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
        dsb(ish);
}

/*
 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
 * necessarily a performance improvement.
 */
#define MAX_DVM_OPS        PTRS_PER_PTE

/*
 * __flush_tlb_range_op - Perform TLBI operation upon a range
 *
 * @op:        TLBI instruction that operates on a range (has 'r' prefix)
 * @start:        The start address of the range
 * @pages:        Range as the number of pages from 'start'
 * @stride:        Flush granularity
 * @asid:        The ASID of the task (0 for IPA instructions)
 * @tlb_level:        Translation Table level hint, if known
 * @tlbi_user:        If 'true', call an additional __tlbi_user()
 *              (typically for user ASIDs). 'flase' for IPA instructions
 * @lpa2:        If 'true', the lpa2 scheme is used as set out below
 *
 * When the CPU does not support TLB range operations, flush the TLB
 * entries one by one at the granularity of 'stride'. If the TLB
 * range ops are supported, then:
 *
 * 1. If FEAT_LPA2 is in use, the start address of a range operation must be
 *    64KB aligned, so flush pages one by one until the alignment is reached
 *    using the non-range operations. This step is skipped if LPA2 is not in
 *    use.
 *
 * 2. The minimum range granularity is decided by 'scale', so multiple range
 *    TLBI operations may be required. Start from scale = 3, flush the largest
 *    possible number of pages ((num+1)*2^(5*scale+1)) that fit into the
 *    requested range, then decrement scale and continue until one or zero pages
 *    are left. We must start from highest scale to ensure 64KB start alignment
 *    is maintained in the LPA2 case.
 *
 * 3. If there is 1 page remaining, flush it through non-range operations. Range
 *    operations can only span an even number of pages. We save this for last to
 *    ensure 64KB start alignment is maintained for the LPA2 case.
 */
#define __flush_tlb_range_op(op, start, pages, stride,                        \
                                asid, tlb_level, tlbi_user, lpa2)        \
do {                                                                        \
        typeof(start) __flush_start = start;                                \
        typeof(pages) __flush_pages = pages;                                \
        int num = 0;                                                        \
        int scale = 3;                                                        \
        int shift = lpa2 ? 16 : PAGE_SHIFT;                                \
        unsigned long addr;                                                \
                                                                        \
        while (__flush_pages > 0) {                                        \
                if (!system_supports_tlb_range() ||                        \
                    __flush_pages == 1 ||                                \
                    (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) {        \
                        addr = __TLBI_VADDR(__flush_start, asid);        \
                        __tlbi_level(op, addr, tlb_level);                \
                        if (tlbi_user)                                        \
                                __tlbi_user_level(op, addr, tlb_level);        \
                        __flush_start += stride;                        \
                        __flush_pages -= stride >> PAGE_SHIFT;                \
                        continue;                                        \
                }                                                        \
                                                                        \
                num = __TLBI_RANGE_NUM(__flush_pages, scale);                \
                if (num >= 0) {                                                \
                        addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \
                                                scale, num, tlb_level);        \
                        __tlbi(r##op, addr);                                \
                        if (tlbi_user)                                        \
                                __tlbi_user(r##op, addr);                \
                        __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \
                        __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\
                }                                                        \
                scale--;                                                \
        }                                                                \
} while (0)

#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
        __flush_tlb_range_op(op, start, pages, stride, 0, tlb_level, false, kvm_lpa2_is_enabled());

static inline bool __flush_tlb_range_limit_excess(unsigned long start,
                unsigned long end, unsigned long pages, unsigned long stride)
{
        /*
         * When the system does not support TLB range based flush
         * operation, (MAX_DVM_OPS - 1) pages can be handled. But
         * with TLB range based operation, MAX_TLBI_RANGE_PAGES
         * pages can be handled.
         */
        if ((!system_supports_tlb_range() &&
             (end - start) >= (MAX_DVM_OPS * stride)) ||
            pages > MAX_TLBI_RANGE_PAGES)
                return true;

        return false;
}

static inline void __flush_tlb_range_nosync(struct mm_struct *mm,
                                     unsigned long start, unsigned long end,
                                     unsigned long stride, bool last_level,
                                     int tlb_level)
{
        unsigned long asid, pages;

        start = round_down(start, stride);
        end = round_up(end, stride);
        pages = (end - start) >> PAGE_SHIFT;

        if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
                flush_tlb_mm(mm);
                return;
        }

        dsb(ishst);
        asid = ASID(mm);

        if (last_level)
                __flush_tlb_range_op(vale1is, start, pages, stride, asid,
                                     tlb_level, true, lpa2_is_enabled());
        else
                __flush_tlb_range_op(vae1is, start, pages, stride, asid,
                                     tlb_level, true, lpa2_is_enabled());

        mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end);
}

static inline void __flush_tlb_range(struct vm_area_struct *vma,
                                     unsigned long start, unsigned long end,
                                     unsigned long stride, bool last_level,
                                     int tlb_level)
{
        __flush_tlb_range_nosync(vma->vm_mm, start, end, stride,
                                 last_level, tlb_level);
        dsb(ish);
}

static inline void flush_tlb_range(struct vm_area_struct *vma,
                                   unsigned long start, unsigned long end)
{
        /*
         * We cannot use leaf-only invalidation here, since we may be invalidating
         * table entries as part of collapsing hugepages or moving page tables.
         * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough
         * information here.
         */
        __flush_tlb_range(vma, start, end, PAGE_SIZE, false, TLBI_TTL_UNKNOWN);
}

static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
        const unsigned long stride = PAGE_SIZE;
        unsigned long pages;

        start = round_down(start, stride);
        end = round_up(end, stride);
        pages = (end - start) >> PAGE_SHIFT;

        if (__flush_tlb_range_limit_excess(start, end, pages, stride)) {
                flush_tlb_all();
                return;
        }

        dsb(ishst);
        __flush_tlb_range_op(vaale1is, start, pages, stride, 0,
                             TLBI_TTL_UNKNOWN, false, lpa2_is_enabled());
        dsb(ish);
        isb();
}

/*
 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
 * table levels (pgd/pud/pmd).
 */
static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
{
        unsigned long addr = __TLBI_VADDR(kaddr, 0);

        dsb(ishst);
        __tlbi(vaae1is, addr);
        dsb(ish);
        isb();
}

static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *batch,
                struct mm_struct *mm, unsigned long start, unsigned long end)
{
        __flush_tlb_range_nosync(mm, start, end, PAGE_SIZE, true, 3);
}
#endif

#endif






























































   55 




















   55 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef IOPRIO_H
#define IOPRIO_H

#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/iocontext.h>

#include <uapi/linux/ioprio.h>

/*
 * Default IO priority.
 */
#define IOPRIO_DEFAULT        IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0)

/*
 * Check that a priority value has a valid class.
 */
static inline bool ioprio_valid(unsigned short ioprio)
{
        unsigned short class = IOPRIO_PRIO_CLASS(ioprio);

        return class > IOPRIO_CLASS_NONE && class <= IOPRIO_CLASS_IDLE;
}

/*
 * if process has set io priority explicitly, use that. if not, convert
 * the cpu scheduler nice value to an io priority
 */
static inline int task_nice_ioprio(struct task_struct *task)
{
        return (task_nice(task) + 20) / 5;
}

/*
 * This is for the case where the task hasn't asked for a specific IO class.
 * Check for idle and rt task process, and return appropriate IO class.
 */
static inline int task_nice_ioclass(struct task_struct *task)
{
        if (task->policy == SCHED_IDLE)
                return IOPRIO_CLASS_IDLE;
        else if (rt_or_dl_task_policy(task))
                return IOPRIO_CLASS_RT;
        else
                return IOPRIO_CLASS_BE;
}

#ifdef CONFIG_BLOCK
/*
 * If the task has set an I/O priority, use that. Otherwise, return
 * the default I/O priority.
 *
 * Expected to be called for current task or with task_lock() held to keep
 * io_context stable.
 */
static inline int __get_task_ioprio(struct task_struct *p)
{
        struct io_context *ioc = p->io_context;
        int prio;

        if (!ioc)
                return IOPRIO_DEFAULT;

        if (p != current)
                lockdep_assert_held(&p->alloc_lock);

        prio = ioc->ioprio;
        if (IOPRIO_PRIO_CLASS(prio) == IOPRIO_CLASS_NONE)
                prio = IOPRIO_PRIO_VALUE(task_nice_ioclass(p),
                                         task_nice_ioprio(p));
        return prio;
}
#else
static inline int __get_task_ioprio(struct task_struct *p)
{
        return IOPRIO_DEFAULT;
}
#endif /* CONFIG_BLOCK */

static inline int get_current_ioprio(void)
{
        return __get_task_ioprio(current);
}

extern int set_task_ioprio(struct task_struct *task, int ioprio);

#ifdef CONFIG_BLOCK
extern int ioprio_check_cap(int ioprio);
#else
static inline int ioprio_check_cap(int ioprio)
{
        return -ENOTBLK;
}
#endif /* CONFIG_BLOCK */

#endif



























































































































































































































































































































































































    5 


    5 

    2 


    2 



    1 





































































































    4 









    1 


    1 








    1 


    1 

















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/slab.h>
#include <linux/stat.h>
#include <linux/sched/xacct.h>
#include <linux/fcntl.h>
#include <linux/file.h>
#include <linux/uio.h>
#include <linux/fsnotify.h>
#include <linux/security.h>
#include <linux/export.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/splice.h>
#include <linux/compat.h>
#include <linux/mount.h>
#include <linux/fs.h>
#include <linux/dax.h>
#include <linux/overflow.h>
#include "internal.h"

#include <linux/uaccess.h>
#include <asm/unistd.h>

/*
 * Performs necessary checks before doing a clone.
 *
 * Can adjust amount of bytes to clone via @req_count argument.
 * Returns appropriate error code that caller should return or
 * zero in case the clone should be allowed.
 */
static int generic_remap_checks(struct file *file_in, loff_t pos_in,
                                struct file *file_out, loff_t pos_out,
                                loff_t *req_count, unsigned int remap_flags)
{
        struct inode *inode_in = file_in->f_mapping->host;
        struct inode *inode_out = file_out->f_mapping->host;
        uint64_t count = *req_count;
        uint64_t bcount;
        loff_t size_in, size_out;
        loff_t bs = inode_out->i_sb->s_blocksize;
        int ret;

        /* The start of both ranges must be aligned to an fs block. */
        if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs))
                return -EINVAL;

        /* Ensure offsets don't wrap. */
        if (pos_in + count < pos_in || pos_out + count < pos_out)
                return -EINVAL;

        size_in = i_size_read(inode_in);
        size_out = i_size_read(inode_out);

        /* Dedupe requires both ranges to be within EOF. */
        if ((remap_flags & REMAP_FILE_DEDUP) &&
            (pos_in >= size_in || pos_in + count > size_in ||
             pos_out >= size_out || pos_out + count > size_out))
                return -EINVAL;

        /* Ensure the infile range is within the infile. */
        if (pos_in >= size_in)
                return -EINVAL;
        count = min(count, size_in - (uint64_t)pos_in);

        ret = generic_write_check_limits(file_out, pos_out, &count);
        if (ret)
                return ret;

        /*
         * If the user wanted us to link to the infile's EOF, round up to the
         * next block boundary for this check.
         *
         * Otherwise, make sure the count is also block-aligned, having
         * already confirmed the starting offsets' block alignment.
         */
        if (pos_in + count == size_in &&
            (!(remap_flags & REMAP_FILE_DEDUP) || pos_out + count == size_out)) {
                bcount = ALIGN(size_in, bs) - pos_in;
        } else {
                if (!IS_ALIGNED(count, bs))
                        count = ALIGN_DOWN(count, bs);
                bcount = count;
        }

        /* Don't allow overlapped cloning within the same file. */
        if (inode_in == inode_out &&
            pos_out + bcount > pos_in &&
            pos_out < pos_in + bcount)
                return -EINVAL;

        /*
         * We shortened the request but the caller can't deal with that, so
         * bounce the request back to userspace.
         */
        if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN))
                return -EINVAL;

        *req_count = count;
        return 0;
}

int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write)
{
        int mask = write ? MAY_WRITE : MAY_READ;
        loff_t tmp;
        int ret;

        if (unlikely(pos < 0 || len < 0))
                return -EINVAL;

        if (unlikely(check_add_overflow(pos, len, &tmp)))
                return -EINVAL;

        ret = security_file_permission(file, mask);
        if (ret)
                return ret;

        return fsnotify_file_area_perm(file, mask, &pos, len);
}
EXPORT_SYMBOL_GPL(remap_verify_area);

/*
 * Ensure that we don't remap a partial EOF block in the middle of something
 * else.  Assume that the offsets have already been checked for block
 * alignment.
 *
 * For clone we only link a partial EOF block above or at the destination file's
 * EOF.  For deduplication we accept a partial EOF block only if it ends at the
 * destination file's EOF (can not link it into the middle of a file).
 *
 * Shorten the request if possible.
 */
static int generic_remap_check_len(struct inode *inode_in,
                                   struct inode *inode_out,
                                   loff_t pos_out,
                                   loff_t *len,
                                   unsigned int remap_flags)
{
        u64 blkmask = i_blocksize(inode_in) - 1;
        loff_t new_len = *len;

        if ((*len & blkmask) == 0)
                return 0;

        if (pos_out + *len < i_size_read(inode_out))
                new_len &= ~blkmask;

        if (new_len == *len)
                return 0;

        if (remap_flags & REMAP_FILE_CAN_SHORTEN) {
                *len = new_len;
                return 0;
        }

        return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL;
}

/* Read a page's worth of file data into the page cache. */
static struct folio *vfs_dedupe_get_folio(struct file *file, loff_t pos)
{
        return read_mapping_folio(file->f_mapping, pos >> PAGE_SHIFT, file);
}

/*
 * Lock two folios, ensuring that we lock in offset order if the folios
 * are from the same file.
 */
static void vfs_lock_two_folios(struct folio *folio1, struct folio *folio2)
{
        /* Always lock in order of increasing index. */
        if (folio1->index > folio2->index)
                swap(folio1, folio2);

        folio_lock(folio1);
        if (folio1 != folio2)
                folio_lock(folio2);
}

/* Unlock two folios, being careful not to unlock the same folio twice. */
static void vfs_unlock_two_folios(struct folio *folio1, struct folio *folio2)
{
        folio_unlock(folio1);
        if (folio1 != folio2)
                folio_unlock(folio2);
}

/*
 * Compare extents of two files to see if they are the same.
 * Caller must have locked both inodes to prevent write races.
 */
static int vfs_dedupe_file_range_compare(struct file *src, loff_t srcoff,
                                         struct file *dest, loff_t dstoff,
                                         loff_t len, bool *is_same)
{
        bool same = true;
        int error = -EINVAL;

        while (len) {
                struct folio *src_folio, *dst_folio;
                void *src_addr, *dst_addr;
                loff_t cmp_len = min(PAGE_SIZE - offset_in_page(srcoff),
                                     PAGE_SIZE - offset_in_page(dstoff));

                cmp_len = min(cmp_len, len);
                if (cmp_len <= 0)
                        goto out_error;

                src_folio = vfs_dedupe_get_folio(src, srcoff);
                if (IS_ERR(src_folio)) {
                        error = PTR_ERR(src_folio);
                        goto out_error;
                }
                dst_folio = vfs_dedupe_get_folio(dest, dstoff);
                if (IS_ERR(dst_folio)) {
                        error = PTR_ERR(dst_folio);
                        folio_put(src_folio);
                        goto out_error;
                }

                vfs_lock_two_folios(src_folio, dst_folio);

                /*
                 * Now that we've locked both folios, make sure they're still
                 * mapped to the file data we're interested in.  If not,
                 * someone is invalidating pages on us and we lose.
                 */
                if (!folio_test_uptodate(src_folio) || !folio_test_uptodate(dst_folio) ||
                    src_folio->mapping != src->f_mapping ||
                    dst_folio->mapping != dest->f_mapping) {
                        same = false;
                        goto unlock;
                }

                src_addr = kmap_local_folio(src_folio,
                                        offset_in_folio(src_folio, srcoff));
                dst_addr = kmap_local_folio(dst_folio,
                                        offset_in_folio(dst_folio, dstoff));

                flush_dcache_folio(src_folio);
                flush_dcache_folio(dst_folio);

                if (memcmp(src_addr, dst_addr, cmp_len))
                        same = false;

                kunmap_local(dst_addr);
                kunmap_local(src_addr);
unlock:
                vfs_unlock_two_folios(src_folio, dst_folio);
                folio_put(dst_folio);
                folio_put(src_folio);

                if (!same)
                        break;

                srcoff += cmp_len;
                dstoff += cmp_len;
                len -= cmp_len;
        }

        *is_same = same;
        return 0;

out_error:
        return error;
}

/*
 * Check that the two inodes are eligible for cloning, the ranges make
 * sense, and then flush all dirty data.  Caller must ensure that the
 * inodes have been locked against any other modifications.
 *
 * If there's an error, then the usual negative error code is returned.
 * Otherwise returns 0 with *len set to the request length.
 */
int
__generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                struct file *file_out, loff_t pos_out,
                                loff_t *len, unsigned int remap_flags,
                                const struct iomap_ops *dax_read_ops)
{
        struct inode *inode_in = file_inode(file_in);
        struct inode *inode_out = file_inode(file_out);
        bool same_inode = (inode_in == inode_out);
        int ret;

        /* Don't touch certain kinds of inodes */
        if (IS_IMMUTABLE(inode_out))
                return -EPERM;

        if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
                return -ETXTBSY;

        /* Don't reflink dirs, pipes, sockets... */
        if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
                return -EISDIR;
        if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
                return -EINVAL;

        /* Zero length dedupe exits immediately; reflink goes to EOF. */
        if (*len == 0) {
                loff_t isize = i_size_read(inode_in);

                if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize)
                        return 0;
                if (pos_in > isize)
                        return -EINVAL;
                *len = isize - pos_in;
                if (*len == 0)
                        return 0;
        }

        /* Check that we don't violate system file offset limits. */
        ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len,
                        remap_flags);
        if (ret || *len == 0)
                return ret;

        /* Wait for the completion of any pending IOs on both files */
        inode_dio_wait(inode_in);
        if (!same_inode)
                inode_dio_wait(inode_out);

        ret = filemap_write_and_wait_range(inode_in->i_mapping,
                        pos_in, pos_in + *len - 1);
        if (ret)
                return ret;

        ret = filemap_write_and_wait_range(inode_out->i_mapping,
                        pos_out, pos_out + *len - 1);
        if (ret)
                return ret;

        /*
         * Check that the extents are the same.
         */
        if (remap_flags & REMAP_FILE_DEDUP) {
                bool                is_same = false;

                if (!IS_DAX(inode_in))
                        ret = vfs_dedupe_file_range_compare(file_in, pos_in,
                                        file_out, pos_out, *len, &is_same);
                else if (dax_read_ops)
                        ret = dax_dedupe_file_range_compare(inode_in, pos_in,
                                        inode_out, pos_out, *len, &is_same,
                                        dax_read_ops);
                else
                        return -EINVAL;
                if (ret)
                        return ret;
                if (!is_same)
                        return -EBADE;
        }

        ret = generic_remap_check_len(inode_in, inode_out, pos_out, len,
                        remap_flags);
        if (ret || *len == 0)
                return ret;

        /* If can't alter the file contents, we're done. */
        if (!(remap_flags & REMAP_FILE_DEDUP))
                ret = file_modified(file_out);

        return ret;
}

int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in,
                                  struct file *file_out, loff_t pos_out,
                                  loff_t *len, unsigned int remap_flags)
{
        return __generic_remap_file_range_prep(file_in, pos_in, file_out,
                                               pos_out, len, remap_flags, NULL);
}
EXPORT_SYMBOL(generic_remap_file_range_prep);

loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in,
                            struct file *file_out, loff_t pos_out,
                            loff_t len, unsigned int remap_flags)
{
        loff_t ret;

        WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP);

        if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
                return -EXDEV;

        ret = generic_file_rw_checks(file_in, file_out);
        if (ret < 0)
                return ret;

        if (!file_in->f_op->remap_file_range)
                return -EOPNOTSUPP;

        ret = remap_verify_area(file_in, pos_in, len, false);
        if (ret)
                return ret;

        ret = remap_verify_area(file_out, pos_out, len, true);
        if (ret)
                return ret;

        file_start_write(file_out);
        ret = file_in->f_op->remap_file_range(file_in, pos_in,
                        file_out, pos_out, len, remap_flags);
        file_end_write(file_out);
        if (ret < 0)
                return ret;

        fsnotify_access(file_in);
        fsnotify_modify(file_out);
        return ret;
}
EXPORT_SYMBOL(vfs_clone_file_range);

/* Check whether we are allowed to dedupe the destination file */
static bool may_dedupe_file(struct file *file)
{
        struct mnt_idmap *idmap = file_mnt_idmap(file);
        struct inode *inode = file_inode(file);

        if (capable(CAP_SYS_ADMIN))
                return true;
        if (file->f_mode & FMODE_WRITE)
                return true;
        if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid()))
                return true;
        if (!inode_permission(idmap, inode, MAY_WRITE))
                return true;
        return false;
}

loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos,
                                 struct file *dst_file, loff_t dst_pos,
                                 loff_t len, unsigned int remap_flags)
{
        loff_t ret;

        WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP |
                                     REMAP_FILE_CAN_SHORTEN));

        /*
         * This is redundant if called from vfs_dedupe_file_range(), but other
         * callers need it and it's not performance sesitive...
         */
        ret = remap_verify_area(src_file, src_pos, len, false);
        if (ret)
                return ret;

        ret = remap_verify_area(dst_file, dst_pos, len, true);
        if (ret)
                return ret;

        /*
         * This needs to be called after remap_verify_area() because of
         * sb_start_write() and before may_dedupe_file() because the mount's
         * MAY_WRITE need to be checked with mnt_get_write_access_file() held.
         */
        ret = mnt_want_write_file(dst_file);
        if (ret)
                return ret;

        ret = -EPERM;
        if (!may_dedupe_file(dst_file))
                goto out_drop_write;

        ret = -EXDEV;
        if (file_inode(src_file)->i_sb != file_inode(dst_file)->i_sb)
                goto out_drop_write;

        ret = -EISDIR;
        if (S_ISDIR(file_inode(dst_file)->i_mode))
                goto out_drop_write;

        ret = -EINVAL;
        if (!dst_file->f_op->remap_file_range)
                goto out_drop_write;

        if (len == 0) {
                ret = 0;
                goto out_drop_write;
        }

        ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file,
                        dst_pos, len, remap_flags | REMAP_FILE_DEDUP);
out_drop_write:
        mnt_drop_write_file(dst_file);

        return ret;
}
EXPORT_SYMBOL(vfs_dedupe_file_range_one);

int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same)
{
        struct file_dedupe_range_info *info;
        struct inode *src = file_inode(file);
        u64 off;
        u64 len;
        int i;
        int ret;
        u16 count = same->dest_count;
        loff_t deduped;

        if (!(file->f_mode & FMODE_READ))
                return -EINVAL;

        if (same->reserved1 || same->reserved2)
                return -EINVAL;

        off = same->src_offset;
        len = same->src_length;

        if (S_ISDIR(src->i_mode))
                return -EISDIR;

        if (!S_ISREG(src->i_mode))
                return -EINVAL;

        if (!file->f_op->remap_file_range)
                return -EOPNOTSUPP;

        ret = remap_verify_area(file, off, len, false);
        if (ret < 0)
                return ret;
        ret = 0;

        if (off + len > i_size_read(src))
                return -EINVAL;

        /* Arbitrary 1G limit on a single dedupe request, can be raised. */
        len = min_t(u64, len, 1 << 30);

        /* pre-format output fields to sane values */
        for (i = 0; i < count; i++) {
                same->info[i].bytes_deduped = 0ULL;
                same->info[i].status = FILE_DEDUPE_RANGE_SAME;
        }

        for (i = 0, info = same->info; i < count; i++, info++) {
                CLASS(fd, dst_fd)(info->dest_fd);

                if (fd_empty(dst_fd)) {
                        info->status = -EBADF;
                        goto next_loop;
                }

                if (info->reserved) {
                        info->status = -EINVAL;
                        goto next_loop;
                }

                deduped = vfs_dedupe_file_range_one(file, off, fd_file(dst_fd),
                                                    info->dest_offset, len,
                                                    REMAP_FILE_CAN_SHORTEN);
                if (deduped == -EBADE)
                        info->status = FILE_DEDUPE_RANGE_DIFFERS;
                else if (deduped < 0)
                        info->status = deduped;
                else
                        info->bytes_deduped = len;

next_loop:
                if (fatal_signal_pending(current))
                        break;
        }
        return ret;
}
EXPORT_SYMBOL(vfs_dedupe_file_range);




































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _DELAYED_CALL_H
#define _DELAYED_CALL_H

/*
 * Poor man's closures; I wish we could've done them sanely polymorphic,
 * but...
 */

struct delayed_call {
        void (*fn)(void *);
        void *arg;
};

#define DEFINE_DELAYED_CALL(name) struct delayed_call name = {NULL, NULL}

/* I really wish we had closures with sane typechecking... */
static inline void set_delayed_call(struct delayed_call *call,
                void (*fn)(void *), void *arg)
{
        call->fn = fn;
        call->arg = arg;
}

static inline void do_delayed_call(struct delayed_call *call)
{
        if (call->fn)
                call->fn(call->arg);
}

static inline void clear_delayed_call(struct delayed_call *call)
{
        call->fn = NULL;
}
#endif
































    3 
    3 



    3 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
// SPDX-License-Identifier: GPL-2.0-only
/*
 *        STP SAP demux
 *
 *        Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 */
#include <linux/mutex.h>
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/llc.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/llc.h>
#include <net/llc_pdu.h>
#include <net/stp.h>

/* 01:80:c2:00:00:20 - 01:80:c2:00:00:2F */
#define GARP_ADDR_MIN        0x20
#define GARP_ADDR_MAX        0x2F
#define GARP_ADDR_RANGE        (GARP_ADDR_MAX - GARP_ADDR_MIN)

static const struct stp_proto __rcu *garp_protos[GARP_ADDR_RANGE + 1] __read_mostly;
static const struct stp_proto __rcu *stp_proto __read_mostly;

static struct llc_sap *sap __read_mostly;
static unsigned int sap_registered;
static DEFINE_MUTEX(stp_proto_mutex);

/* Called under rcu_read_lock from LLC */
static int stp_pdu_rcv(struct sk_buff *skb, struct net_device *dev,
                       struct packet_type *pt, struct net_device *orig_dev)
{
        const struct ethhdr *eh = eth_hdr(skb);
        const struct llc_pdu_un *pdu = llc_pdu_un_hdr(skb);
        const struct stp_proto *proto;

        if (pdu->ssap != LLC_SAP_BSPAN ||
            pdu->dsap != LLC_SAP_BSPAN ||
            pdu->ctrl_1 != LLC_PDU_TYPE_U)
                goto err;

        if (eh->h_dest[5] >= GARP_ADDR_MIN && eh->h_dest[5] <= GARP_ADDR_MAX) {
                proto = rcu_dereference(garp_protos[eh->h_dest[5] -
                                                    GARP_ADDR_MIN]);
                if (proto &&
                    !ether_addr_equal(eh->h_dest, proto->group_address))
                        goto err;
        } else
                proto = rcu_dereference(stp_proto);

        if (!proto)
                goto err;

        proto->rcv(proto, skb, dev);
        return 0;

err:
        kfree_skb(skb);
        return 0;
}

int stp_proto_register(const struct stp_proto *proto)
{
        int err = 0;

        mutex_lock(&stp_proto_mutex);
        if (sap_registered++ == 0) {
                sap = llc_sap_open(LLC_SAP_BSPAN, stp_pdu_rcv);
                if (!sap) {
                        err = -ENOMEM;
                        goto out;
                }
        }
        if (is_zero_ether_addr(proto->group_address))
                rcu_assign_pointer(stp_proto, proto);
        else
                rcu_assign_pointer(garp_protos[proto->group_address[5] -
                                               GARP_ADDR_MIN], proto);
out:
        mutex_unlock(&stp_proto_mutex);
        return err;
}
EXPORT_SYMBOL_GPL(stp_proto_register);

void stp_proto_unregister(const struct stp_proto *proto)
{
        mutex_lock(&stp_proto_mutex);
        if (is_zero_ether_addr(proto->group_address))
                RCU_INIT_POINTER(stp_proto, NULL);
        else
                RCU_INIT_POINTER(garp_protos[proto->group_address[5] -
                                               GARP_ADDR_MIN], NULL);
        synchronize_rcu();

        if (--sap_registered == 0)
                llc_sap_put(sap);
        mutex_unlock(&stp_proto_mutex);
}
EXPORT_SYMBOL_GPL(stp_proto_unregister);

MODULE_DESCRIPTION("SAP demux for IEEE 802.1D Spanning Tree Protocol (STP)");
MODULE_LICENSE("GPL");
































































































   37 


   37 
   37 
   37 





















































































































































































  636 



  632 




  247 
  621 


  634 





  635 




















  505 


  505 


  506 































































  621 










  620 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
// SPDX-License-Identifier: GPL-2.0
/*
 *  mm/pgtable-generic.c
 *
 *  Generic pgtable methods declared in linux/pgtable.h
 *
 *  Copyright (C) 2010  Linus Torvalds
 */

#include <linux/pagemap.h>
#include <linux/hugetlb.h>
#include <linux/pgtable.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mm_inline.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>

/*
 * If a p?d_bad entry is found while walking page tables, report
 * the error, before resetting entry to p?d_none.  Usually (but
 * very seldom) called out from the p?d_none_or_clear_bad macros.
 */

void pgd_clear_bad(pgd_t *pgd)
{
        pgd_ERROR(*pgd);
        pgd_clear(pgd);
}

#ifndef __PAGETABLE_P4D_FOLDED
void p4d_clear_bad(p4d_t *p4d)
{
        p4d_ERROR(*p4d);
        p4d_clear(p4d);
}
#endif

#ifndef __PAGETABLE_PUD_FOLDED
void pud_clear_bad(pud_t *pud)
{
        pud_ERROR(*pud);
        pud_clear(pud);
}
#endif

/*
 * Note that the pmd variant below can't be stub'ed out just as for p4d/pud
 * above. pmd folding is special and typically pmd_* macros refer to upper
 * level even when folded
 */
void pmd_clear_bad(pmd_t *pmd)
{
        pmd_ERROR(*pmd);
        pmd_clear(pmd);
}

#ifndef __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
/*
 * Only sets the access flags (dirty, accessed), as well as write
 * permission. Furthermore, we know it always gets set to a "more
 * permissive" setting, which allows most architectures to optimize
 * this. We return whether the PTE actually changed, which in turn
 * instructs the caller to do things like update__mmu_cache.  This
 * used to be done in the caller, but sparc needs minor faults to
 * force that call on sun4c so we changed this macro slightly
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
                          unsigned long address, pte_t *ptep,
                          pte_t entry, int dirty)
{
        int changed = !pte_same(ptep_get(ptep), entry);
        if (changed) {
                set_pte_at(vma->vm_mm, address, ptep, entry);
                flush_tlb_fix_spurious_fault(vma, address, ptep);
        }
        return changed;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
int ptep_clear_flush_young(struct vm_area_struct *vma,
                           unsigned long address, pte_t *ptep)
{
        int young;
        young = ptep_test_and_clear_young(vma, address, ptep);
        if (young)
                flush_tlb_page(vma, address);
        return young;
}
#endif

#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
                       pte_t *ptep)
{
        struct mm_struct *mm = (vma)->vm_mm;
        pte_t pte;
        pte = ptep_get_and_clear(mm, address, ptep);
        if (pte_accessible(mm, pte))
                flush_tlb_page(vma, address);
        return pte;
}
#endif

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

#ifndef __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
int pmdp_set_access_flags(struct vm_area_struct *vma,
                          unsigned long address, pmd_t *pmdp,
                          pmd_t entry, int dirty)
{
        int changed = !pmd_same(*pmdp, entry);
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        if (changed) {
                set_pmd_at(vma->vm_mm, address, pmdp, entry);
                flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        }
        return changed;
}
#endif

#ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
int pmdp_clear_flush_young(struct vm_area_struct *vma,
                           unsigned long address, pmd_t *pmdp)
{
        int young;
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        young = pmdp_test_and_clear_young(vma, address, pmdp);
        if (young)
                flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return young;
}
#endif

#ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
                            pmd_t *pmdp)
{
        pmd_t pmd;
        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) &&
                           !pmd_devmap(*pmdp));
        pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);
        flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return pmd;
}

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address,
                            pud_t *pudp)
{
        pud_t pud;

        VM_BUG_ON(address & ~HPAGE_PUD_MASK);
        VM_BUG_ON(!pud_trans_huge(*pudp) && !pud_devmap(*pudp));
        pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp);
        flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE);
        return pud;
}
#endif
#endif

#ifndef __HAVE_ARCH_PGTABLE_DEPOSIT
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
                                pgtable_t pgtable)
{
        assert_spin_locked(pmd_lockptr(mm, pmdp));

        /* FIFO */
        if (!pmd_huge_pte(mm, pmdp))
                INIT_LIST_HEAD(&pgtable->lru);
        else
                list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
        pmd_huge_pte(mm, pmdp) = pgtable;
}
#endif

#ifndef __HAVE_ARCH_PGTABLE_WITHDRAW
/* no "address" argument so destroys page coloring of some arch */
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
        pgtable_t pgtable;

        assert_spin_locked(pmd_lockptr(mm, pmdp));

        /* FIFO */
        pgtable = pmd_huge_pte(mm, pmdp);
        pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
                                                          struct page, lru);
        if (pmd_huge_pte(mm, pmdp))
                list_del(&pgtable->lru);
        return pgtable;
}
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE
pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
                     pmd_t *pmdp)
{
        VM_WARN_ON_ONCE(!pmd_present(*pmdp));
        pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp));
        flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return old;
}
#endif

#ifndef __HAVE_ARCH_PMDP_INVALIDATE_AD
pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address,
                         pmd_t *pmdp)
{
        VM_WARN_ON_ONCE(!pmd_present(*pmdp));
        return pmdp_invalidate(vma, address, pmdp);
}
#endif

#ifndef pmdp_collapse_flush
pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
                          pmd_t *pmdp)
{
        /*
         * pmd and hugepage pte format are same. So we could
         * use the same function.
         */
        pmd_t pmd;

        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
        VM_BUG_ON(pmd_trans_huge(*pmdp));
        pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp);

        /* collapse entails shooting down ptes not pmd */
        flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
        return pmd;
}
#endif

/* arch define pte_free_defer in asm/pgalloc.h for its own implementation */
#ifndef pte_free_defer
static void pte_free_now(struct rcu_head *head)
{
        struct page *page;

        page = container_of(head, struct page, rcu_head);
        pte_free(NULL /* mm not passed and not used */, (pgtable_t)page);
}

void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
{
        struct page *page;

        page = pgtable;
        call_rcu(&page->rcu_head, pte_free_now);
}
#endif /* pte_free_defer */
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#if defined(CONFIG_GUP_GET_PXX_LOW_HIGH) && \
        (defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RCU))
/*
 * See the comment above ptep_get_lockless() in include/linux/pgtable.h:
 * the barriers in pmdp_get_lockless() cannot guarantee that the value in
 * pmd_high actually belongs with the value in pmd_low; but holding interrupts
 * off blocks the TLB flush between present updates, which guarantees that a
 * successful __pte_offset_map() points to a page from matched halves.
 */
static unsigned long pmdp_get_lockless_start(void)
{
        unsigned long irqflags;

        local_irq_save(irqflags);
        return irqflags;
}
static void pmdp_get_lockless_end(unsigned long irqflags)
{
        local_irq_restore(irqflags);
}
#else
static unsigned long pmdp_get_lockless_start(void) { return 0; }
static void pmdp_get_lockless_end(unsigned long irqflags) { }
#endif

pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp)
{
        unsigned long irqflags;
        pmd_t pmdval;

        rcu_read_lock();
        irqflags = pmdp_get_lockless_start();
        pmdval = pmdp_get_lockless(pmd);
        pmdp_get_lockless_end(irqflags);

        if (pmdvalp)
                *pmdvalp = pmdval;
        if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval)))
                goto nomap;
        if (unlikely(pmd_trans_huge(pmdval) || pmd_devmap(pmdval)))
                goto nomap;
        if (unlikely(pmd_bad(pmdval))) {
                pmd_clear_bad(pmd);
                goto nomap;
        }
        return __pte_map(&pmdval, addr);
nomap:
        rcu_read_unlock();
        return NULL;
}

pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd,
                                unsigned long addr, spinlock_t **ptlp)
{
        pmd_t pmdval;
        pte_t *pte;

        pte = __pte_offset_map(pmd, addr, &pmdval);
        if (likely(pte))
                *ptlp = pte_lockptr(mm, &pmdval);
        return pte;
}

pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd,
                                unsigned long addr, pmd_t *pmdvalp,
                                spinlock_t **ptlp)
{
        pte_t *pte;

        VM_WARN_ON_ONCE(!pmdvalp);
        pte = __pte_offset_map(pmd, addr, pmdvalp);
        if (likely(pte))
                *ptlp = pte_lockptr(mm, pmdvalp);
        return pte;
}

/*
 * pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation
 * __pte_offset_map_lock() below, is usually called with the pmd pointer for
 * addr, reached by walking down the mm's pgd, p4d, pud for addr: either while
 * holding mmap_lock or vma lock for read or for write; or in truncate or rmap
 * context, while holding file's i_mmap_lock or anon_vma lock for read (or for
 * write). In a few cases, it may be used with pmd pointing to a pmd_t already
 * copied to or constructed on the stack.
 *
 * When successful, it returns the pte pointer for addr, with its page table
 * kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent
 * modification by software, with a pointer to that spinlock in ptlp (in some
 * configs mm->page_table_lock, in SPLIT_PTLOCK configs a spinlock in table's
 * struct page).  pte_unmap_unlock(pte, ptl) to unlock and unmap afterwards.
 *
 * But it is unsuccessful, returning NULL with *ptlp unchanged, if there is no
 * page table at *pmd: if, for example, the page table has just been removed,
 * or replaced by the huge pmd of a THP.  (When successful, *pmd is rechecked
 * after acquiring the ptlock, and retried internally if it changed: so that a
 * page table can be safely removed or replaced by THP while holding its lock.)
 *
 * pte_offset_map(pmd, addr), and its internal helper __pte_offset_map() above,
 * just returns the pte pointer for addr, its page table kmapped if necessary;
 * or NULL if there is no page table at *pmd.  It does not attempt to lock the
 * page table, so cannot normally be used when the page table is to be updated,
 * or when entries read must be stable.  But it does take rcu_read_lock(): so
 * that even when page table is racily removed, it remains a valid though empty
 * and disconnected table.  Until pte_unmap(pte) unmaps and rcu_read_unlock()s
 * afterwards.
 *
 * pte_offset_map_ro_nolock(mm, pmd, addr, ptlp), above, is like pte_offset_map();
 * but when successful, it also outputs a pointer to the spinlock in ptlp - as
 * pte_offset_map_lock() does, but in this case without locking it.  This helps
 * the caller to avoid a later pte_lockptr(mm, *pmd), which might by that time
 * act on a changed *pmd: pte_offset_map_ro_nolock() provides the correct spinlock
 * pointer for the page table that it returns. Even after grabbing the spinlock,
 * we might be looking either at a page table that is still mapped or one that
 * was unmapped and is about to get freed. But for R/O access this is sufficient.
 * So it is only applicable for read-only cases where any modification operations
 * to the page table are not allowed even if the corresponding spinlock is held
 * afterwards.
 *
 * pte_offset_map_rw_nolock(mm, pmd, addr, pmdvalp, ptlp), above, is like
 * pte_offset_map_ro_nolock(); but when successful, it also outputs the pdmval.
 * It is applicable for may-write cases where any modification operations to the
 * page table may happen after the corresponding spinlock is held afterwards.
 * But the users should make sure the page table is stable like checking pte_same()
 * or checking pmd_same() by using the output pmdval before performing the write
 * operations.
 *
 * Note: "RO" / "RW" expresses the intended semantics, not that the *kmap* will
 * be read-only/read-write protected.
 *
 * Note that free_pgtables(), used after unmapping detached vmas, or when
 * exiting the whole mm, does not take page table lock before freeing a page
 * table, and may not use RCU at all: "outsiders" like khugepaged should avoid
 * pte_offset_map() and co once the vma is detached from mm or mm_users is zero.
 */
pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd,
                             unsigned long addr, spinlock_t **ptlp)
{
        spinlock_t *ptl;
        pmd_t pmdval;
        pte_t *pte;
again:
        pte = __pte_offset_map(pmd, addr, &pmdval);
        if (unlikely(!pte))
                return pte;
        ptl = pte_lockptr(mm, &pmdval);
        spin_lock(ptl);
        if (likely(pmd_same(pmdval, pmdp_get_lockless(pmd)))) {
                *ptlp = ptl;
                return pte;
        }
        pte_unmap_unlock(pte, ptl);
        goto again;
}



















































































































































   29 












 1675 
 1674 




 1669 
 1676 
















































































    8 






















































































































































    8 
 1674 






























































   29 


























































   21 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * Sleepable Read-Copy Update mechanism for mutual exclusion
 *
 * Copyright (C) IBM Corporation, 2006
 * Copyright (C) Fujitsu, 2012
 *
 * Author: Paul McKenney <paulmck@linux.ibm.com>
 *           Lai Jiangshan <laijs@cn.fujitsu.com>
 *
 * For detailed explanation of Read-Copy Update mechanism see -
 *                Documentation/RCU/ *.txt
 *
 */

#ifndef _LINUX_SRCU_H
#define _LINUX_SRCU_H

#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/rcu_segcblist.h>

struct srcu_struct;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
                       struct lock_class_key *key);

#define init_srcu_struct(ssp) \
({ \
        static struct lock_class_key __srcu_key; \
        \
        __init_srcu_struct((ssp), #ssp, &__srcu_key); \
})

#define __SRCU_DEP_MAP_INIT(srcu_name)        .dep_map = { .name = #srcu_name },
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

int init_srcu_struct(struct srcu_struct *ssp);

#define __SRCU_DEP_MAP_INIT(srcu_name)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
#define SRCU_READ_FLAVOR_NORMAL        0x1                // srcu_read_lock().
#define SRCU_READ_FLAVOR_NMI        0x2                // srcu_read_lock_nmisafe().
#define SRCU_READ_FLAVOR_LITE        0x4                // srcu_read_lock_lite().
#define SRCU_READ_FLAVOR_FAST        0x8                // srcu_read_lock_fast().
#define SRCU_READ_FLAVOR_ALL   (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
                                SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above.
#define SRCU_READ_FLAVOR_SLOWGP        (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)
                                                // Flavors requiring synchronize_rcu()
                                                // instead of smp_mb().
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);

#ifdef CONFIG_TINY_SRCU
#include <linux/srcutiny.h>
#elif defined(CONFIG_TREE_SRCU)
#include <linux/srcutree.h>
#else
#error "Unknown SRCU implementation specified to kernel configuration"
#endif

void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
                void (*func)(struct rcu_head *head));
void cleanup_srcu_struct(struct srcu_struct *ssp);
void synchronize_srcu(struct srcu_struct *ssp);

#define SRCU_GET_STATE_COMPLETED 0x1

/**
 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
 *
 * Returns a value that poll_state_synchronize_srcu() will always treat
 * as a cookie whose grace period has already completed.
 */
static inline unsigned long get_completed_synchronize_srcu(void)
{
        return SRCU_GET_STATE_COMPLETED;
}

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);

// Maximum number of unsigned long values corresponding to
// not-yet-completed SRCU grace periods.
#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2

/**
 * same_state_synchronize_srcu - Are two old-state values identical?
 * @oldstate1: First old-state value.
 * @oldstate2: Second old-state value.
 *
 * The two old-state values must have been obtained from either
 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
 * get_completed_synchronize_srcu().  Returns @true if the two values are
 * identical and @false otherwise.  This allows structures whose lifetimes
 * are tracked by old-state values to push these values to a list header,
 * allowing those structures to be slightly smaller.
 */
static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
{
        return oldstate1 == oldstate2;
}

#ifdef CONFIG_NEED_SRCU_NMI_SAFE
int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
#else
static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
{
        return __srcu_read_lock(ssp);
}
static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
{
        __srcu_read_unlock(ssp, idx);
}
#endif /* CONFIG_NEED_SRCU_NMI_SAFE */

void srcu_init(void);

#ifdef CONFIG_DEBUG_LOCK_ALLOC

/**
 * srcu_read_lock_held - might we be in SRCU read-side critical section?
 * @ssp: The srcu_struct structure to check
 *
 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
 * read-side critical section.  In absence of CONFIG_DEBUG_LOCK_ALLOC,
 * this assumes we are in an SRCU read-side critical section unless it can
 * prove otherwise.
 *
 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
 * and while lockdep is disabled.
 *
 * Note that SRCU is based on its own statemachine and it doesn't
 * relies on normal RCU, it can be called from the CPU which
 * is in the idle loop from an RCU point of view or offline.
 */
static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
        if (!debug_lockdep_rcu_enabled())
                return 1;
        return lock_is_held(&ssp->dep_map);
}

/*
 * Annotations provide deadlock detection for SRCU.
 *
 * Similar to other lockdep annotations, except there is an additional
 * srcu_lock_sync(), which is basically an empty *write*-side critical section,
 * see lock_sync() for more information.
 */

/* Annotates a srcu_read_lock() */
static inline void srcu_lock_acquire(struct lockdep_map *map)
{
        lock_map_acquire_read(map);
}

/* Annotates a srcu_read_lock() */
static inline void srcu_lock_release(struct lockdep_map *map)
{
        lock_map_release(map);
}

/* Annotates a synchronize_srcu() */
static inline void srcu_lock_sync(struct lockdep_map *map)
{
        lock_map_sync(map);
}

#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
        return 1;
}

#define srcu_lock_acquire(m) do { } while (0)
#define srcu_lock_release(m) do { } while (0)
#define srcu_lock_sync(m) do { } while (0)

#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */


/**
 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 * @c: condition to check for update-side use
 *
 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
 * critical section will result in an RCU-lockdep splat, unless @c evaluates
 * to 1.  The @c argument will normally be a logical expression containing
 * lockdep_is_held() calls.
 */
#define srcu_dereference_check(p, ssp, c) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
                                (c) || srcu_read_lock_held(ssp), __rcu)

/**
 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 *
 * Makes rcu_dereference_check() do the dirty work.  If PROVE_RCU
 * is enabled, invoking this outside of an RCU read-side critical
 * section will result in an RCU-lockdep splat.
 */
#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)

/**
 * srcu_dereference_notrace - no tracing and no lockdep calls from here
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 */
#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)

/**
 * srcu_read_lock - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section.  Note that SRCU read-side
 * critical sections may be nested.  However, it is illegal to
 * call anything that waits on an SRCU grace period for the same
 * srcu_struct, whether directly or indirectly.  Please note that
 * one way to indirectly wait on an SRCU grace period is to acquire
 * a mutex that is held elsewhere while calling synchronize_srcu() or
 * synchronize_srcu_expedited().
 *
 * The return value from srcu_read_lock() is guaranteed to be
 * non-negative.  This value must be passed unaltered to the matching
 * srcu_read_unlock().  Note that srcu_read_lock() and the matching
 * srcu_read_unlock() must occur in the same context, for example, it is
 * illegal to invoke srcu_read_unlock() in an irq handler if the matching
 * srcu_read_lock() was invoked in process context.  Or, for that matter to
 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
 * from another.
 */
static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        retval = __srcu_read_lock(ssp);
        srcu_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but for a light-weight
 * smp_mb()-free reader.  See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_fast() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.  Note that grace-period auto-expediting is disabled for _fast
 * srcu_struct structures because auto-expedited grace periods invoke
 * synchronize_rcu_expedited(), IPIs and all.
 *
 * Note that srcu_read_lock_fast() can be invoked only from those contexts
 * where RCU is watching, that is, from contexts where it would be legal
 * to invoke rcu_read_lock().  Otherwise, lockdep will complain.
 */
static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)
{
        struct srcu_ctr __percpu *retval;

        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
        retval = __srcu_read_lock_fast(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter a semaphore-like SRCU read-side critical section, but for
 * a light-weight smp_mb()-free reader.  See srcu_read_lock_fast() and
 * srcu_down_read() for more information.
 *
 * The same srcu_struct may be used concurrently by srcu_down_read_fast()
 * and srcu_read_lock_fast().
 */
static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)
{
        WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
        return __srcu_read_lock_fast(ssp);
}

/**
 * srcu_read_lock_lite - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but for a light-weight
 * smp_mb()-free reader.  See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_lite() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.  Note that grace-period auto-expediting is disabled for _lite
 * srcu_struct structures because auto-expedited grace periods invoke
 * synchronize_rcu_expedited(), IPIs and all.
 *
 * Note that srcu_read_lock_lite() can be invoked only from those contexts
 * where RCU is watching, that is, from contexts where it would be legal
 * to invoke rcu_read_lock().  Otherwise, lockdep will complain.
 */
static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_LITE);
        retval = __srcu_read_lock_lite(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but in an NMI-safe manner.
 * See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.
 */
static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
        retval = __srcu_read_lock_nmisafe(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/* Used by tracing, cannot be traced and cannot invoke lockdep. */
static inline notrace int
srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        retval = __srcu_read_lock(ssp);
        return retval;
}

/**
 * srcu_down_read - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter a semaphore-like SRCU read-side critical section.  Note that
 * SRCU read-side critical sections may be nested.  However, it is
 * illegal to call anything that waits on an SRCU grace period for the
 * same srcu_struct, whether directly or indirectly.  Please note that
 * one way to indirectly wait on an SRCU grace period is to acquire
 * a mutex that is held elsewhere while calling synchronize_srcu() or
 * synchronize_srcu_expedited().  But if you want lockdep to help you
 * keep this stuff straight, you should instead use srcu_read_lock().
 *
 * The semaphore-like nature of srcu_down_read() means that the matching
 * srcu_up_read() can be invoked from some other context, for example,
 * from some other task or from an irq handler.  However, neither
 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
 *
 * Calls to srcu_down_read() may be nested, similar to the manner in
 * which calls to down_read() may be nested.  The same srcu_struct may be
 * used concurrently by srcu_down_read() and srcu_read_lock().
 */
static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
{
        WARN_ON_ONCE(in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        return __srcu_read_lock(ssp);
}

/**
 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock().
 *
 * Exit an SRCU read-side critical section.
 */
static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock(ssp, idx);
}

/**
 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @scp: return value from corresponding srcu_read_lock_fast().
 *
 * Exit a light-weight SRCU read-side critical section.
 */
static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
        __releases(ssp)
{
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_fast(ssp, scp);
}

/**
 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @scp: return value from corresponding srcu_read_lock_fast().
 *
 * Exit an SRCU read-side critical section, but not necessarily from
 * the same context as the maching srcu_down_read_fast().
 */
static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
        __releases(ssp)
{
        WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        __srcu_read_unlock_fast(ssp, scp);
}

/**
 * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock_lite().
 *
 * Exit a light-weight SRCU read-side critical section.
 */
static inline void srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_lite(ssp, idx);
}

/**
 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock_nmisafe().
 *
 * Exit an SRCU read-side critical section, but in an NMI-safe manner.
 */
static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
        rcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_nmisafe(ssp, idx);
}

/* Used by tracing, cannot be traced and cannot call lockdep. */
static inline notrace void
srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
{
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        __srcu_read_unlock(ssp, idx);
}

/**
 * srcu_up_read - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock().
 *
 * Exit an SRCU read-side critical section, but not necessarily from
 * the same context as the maching srcu_down_read().
 */
static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        WARN_ON_ONCE(in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        __srcu_read_unlock(ssp, idx);
}

/**
 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
 *
 * Converts the preceding srcu_read_unlock into a two-way memory barrier.
 *
 * Call this after srcu_read_unlock, to guarantee that all memory operations
 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
 * the preceding srcu_read_unlock.
 */
static inline void smp_mb__after_srcu_read_unlock(void)
{
        /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
}

/**
 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
 *
 * Converts the preceding srcu_read_lock into a two-way memory barrier.
 *
 * Call this after srcu_read_lock, to guarantee that all memory operations
 * that occur after smp_mb__after_srcu_read_lock will appear to happen after
 * the preceding srcu_read_lock.
 */
static inline void smp_mb__after_srcu_read_lock(void)
{
        /* __srcu_read_lock has smp_mb() internally so nothing to do here. */
}

DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
                    _T->idx = srcu_read_lock(_T->lock),
                    srcu_read_unlock(_T->lock, _T->idx),
                    int idx)

#endif










































































































































































































































































































































 1795 
 1798 



   29 
   29 



 1797 
 1797 




















































































































































































































































































































































































































































































































 1797 

























  460 
 1795 

























    4 










    4 


























 1020 

















 1020 



 1018 







































































































































































































  275 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * Read-Copy Update mechanism for mutual exclusion
 *
 * Copyright IBM Corporation, 2001
 *
 * Author: Dipankar Sarma <dipankar@in.ibm.com>
 *
 * Based on the original work by Paul McKenney <paulmck@vnet.ibm.com>
 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
 * Papers:
 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf
 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001)
 *
 * For detailed explanation of Read-Copy Update mechanism see -
 *                http://lse.sourceforge.net/locking/rcupdate.html
 *
 */

#ifndef __LINUX_RCUPDATE_H
#define __LINUX_RCUPDATE_H

#include <linux/types.h>
#include <linux/compiler.h>
#include <linux/atomic.h>
#include <linux/irqflags.h>
#include <linux/preempt.h>
#include <linux/bottom_half.h>
#include <linux/lockdep.h>
#include <linux/cleanup.h>
#include <asm/processor.h>
#include <linux/context_tracking_irq.h>

#define ULONG_CMP_GE(a, b)        (ULONG_MAX / 2 >= (a) - (b))
#define ULONG_CMP_LT(a, b)        (ULONG_MAX / 2 < (a) - (b))

#define RCU_SEQ_CTR_SHIFT    2
#define RCU_SEQ_STATE_MASK   ((1 << RCU_SEQ_CTR_SHIFT) - 1)

/* Exported common interfaces */
void call_rcu(struct rcu_head *head, rcu_callback_t func);
void rcu_barrier_tasks(void);
void synchronize_rcu(void);

struct rcu_gp_oldstate;
unsigned long get_completed_synchronize_rcu(void);
void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);

// Maximum number of unsigned long values corresponding to
// not-yet-completed RCU grace periods.
#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2

/**
 * same_state_synchronize_rcu - Are two old-state values identical?
 * @oldstate1: First old-state value.
 * @oldstate2: Second old-state value.
 *
 * The two old-state values must have been obtained from either
 * get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or
 * get_completed_synchronize_rcu().  Returns @true if the two values are
 * identical and @false otherwise.  This allows structures whose lifetimes
 * are tracked by old-state values to push these values to a list header,
 * allowing those structures to be slightly smaller.
 */
static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2)
{
        return oldstate1 == oldstate2;
}

#ifdef CONFIG_PREEMPT_RCU

void __rcu_read_lock(void);
void __rcu_read_unlock(void);

/*
 * Defined as a macro as it is a very low level header included from
 * areas that don't even know about current.  This gives the rcu_read_lock()
 * nesting depth, but makes sense only if CONFIG_PREEMPT_RCU -- in other
 * types of kernel builds, the rcu_read_lock() nesting depth is unknowable.
 */
#define rcu_preempt_depth() READ_ONCE(current->rcu_read_lock_nesting)

#else /* #ifdef CONFIG_PREEMPT_RCU */

#ifdef CONFIG_TINY_RCU
#define rcu_read_unlock_strict() do { } while (0)
#else
void rcu_read_unlock_strict(void);
#endif

static inline void __rcu_read_lock(void)
{
        preempt_disable();
}

static inline void __rcu_read_unlock(void)
{
        if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD))
                rcu_read_unlock_strict();
        preempt_enable();
}

static inline int rcu_preempt_depth(void)
{
        return 0;
}

#endif /* #else #ifdef CONFIG_PREEMPT_RCU */

#ifdef CONFIG_RCU_LAZY
void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func);
#else
static inline void call_rcu_hurry(struct rcu_head *head, rcu_callback_t func)
{
        call_rcu(head, func);
}
#endif

/* Internal to kernel */
void rcu_init(void);
extern int rcu_scheduler_active;
void rcu_sched_clock_irq(int user);

#ifdef CONFIG_RCU_STALL_COMMON
void rcu_sysrq_start(void);
void rcu_sysrq_end(void);
#else /* #ifdef CONFIG_RCU_STALL_COMMON */
static inline void rcu_sysrq_start(void) { }
static inline void rcu_sysrq_end(void) { }
#endif /* #else #ifdef CONFIG_RCU_STALL_COMMON */

#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK))
void rcu_irq_work_resched(void);
#else
static __always_inline void rcu_irq_work_resched(void) { }
#endif

#ifdef CONFIG_RCU_NOCB_CPU
void rcu_init_nohz(void);
int rcu_nocb_cpu_offload(int cpu);
int rcu_nocb_cpu_deoffload(int cpu);
void rcu_nocb_flush_deferred_wakeup(void);

#define RCU_NOCB_LOCKDEP_WARN(c, s) RCU_LOCKDEP_WARN(c, s)

#else /* #ifdef CONFIG_RCU_NOCB_CPU */

static inline void rcu_init_nohz(void) { }
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
static inline void rcu_nocb_flush_deferred_wakeup(void) { }

#define RCU_NOCB_LOCKDEP_WARN(c, s)

#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */

/*
 * Note a quasi-voluntary context switch for RCU-tasks's benefit.
 * This is a macro rather than an inline function to avoid #include hell.
 */
#ifdef CONFIG_TASKS_RCU_GENERIC

# ifdef CONFIG_TASKS_RCU
# define rcu_tasks_classic_qs(t, preempt)                                \
        do {                                                                \
                if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout))        \
                        WRITE_ONCE((t)->rcu_tasks_holdout, false);        \
        } while (0)
void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
void synchronize_rcu_tasks(void);
void rcu_tasks_torture_stats_print(char *tt, char *tf);
# else
# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
# define call_rcu_tasks call_rcu
# define synchronize_rcu_tasks synchronize_rcu
# endif

# ifdef CONFIG_TASKS_TRACE_RCU
// Bits for ->trc_reader_special.b.need_qs field.
#define TRC_NEED_QS                0x1  // Task needs a quiescent state.
#define TRC_NEED_QS_CHECKED        0x2  // Task has been checked for needing quiescent state.

u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new);
void rcu_tasks_trace_qs_blkd(struct task_struct *t);

# define rcu_tasks_trace_qs(t)                                                        \
        do {                                                                        \
                int ___rttq_nesting = READ_ONCE((t)->trc_reader_nesting);        \
                                                                                \
                if (unlikely(READ_ONCE((t)->trc_reader_special.b.need_qs) == TRC_NEED_QS) &&        \
                    likely(!___rttq_nesting)) {                                        \
                        rcu_trc_cmpxchg_need_qs((t), TRC_NEED_QS, TRC_NEED_QS_CHECKED);        \
                } else if (___rttq_nesting && ___rttq_nesting != INT_MIN &&        \
                           !READ_ONCE((t)->trc_reader_special.b.blocked)) {        \
                        rcu_tasks_trace_qs_blkd(t);                                \
                }                                                                \
        } while (0)
void rcu_tasks_trace_torture_stats_print(char *tt, char *tf);
# else
# define rcu_tasks_trace_qs(t) do { } while (0)
# endif

#define rcu_tasks_qs(t, preempt)                                        \
do {                                                                        \
        rcu_tasks_classic_qs((t), (preempt));                                \
        rcu_tasks_trace_qs(t);                                                \
} while (0)

# ifdef CONFIG_TASKS_RUDE_RCU
void synchronize_rcu_tasks_rude(void);
void rcu_tasks_rude_torture_stats_print(char *tt, char *tf);
# endif

#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
void exit_tasks_rcu_start(void);
void exit_tasks_rcu_finish(void);
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
#define rcu_tasks_classic_qs(t, preempt) do { } while (0)
#define rcu_tasks_qs(t, preempt) do { } while (0)
#define rcu_note_voluntary_context_switch(t) do { } while (0)
#define call_rcu_tasks call_rcu
#define synchronize_rcu_tasks synchronize_rcu
static inline void exit_tasks_rcu_start(void) { }
static inline void exit_tasks_rcu_finish(void) { }
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */

/**
 * rcu_trace_implies_rcu_gp - does an RCU Tasks Trace grace period imply an RCU grace period?
 *
 * As an accident of implementation, an RCU Tasks Trace grace period also
 * acts as an RCU grace period.  However, this could change at any time.
 * Code relying on this accident must call this function to verify that
 * this accident is still happening.
 *
 * You have been warned!
 */
static inline bool rcu_trace_implies_rcu_gp(void) { return true; }

/**
 * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
 *
 * This macro resembles cond_resched(), except that it is defined to
 * report potential quiescent states to RCU-tasks even if the cond_resched()
 * machinery were to be shut off, as some advocate for PREEMPTION kernels.
 */
#define cond_resched_tasks_rcu_qs() \
do { \
        rcu_tasks_qs(current, false); \
        cond_resched(); \
} while (0)

/**
 * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states
 * @old_ts: jiffies at start of processing.
 *
 * This helper is for long-running softirq handlers, such as NAPI threads in
 * networking. The caller should initialize the variable passed in as @old_ts
 * at the beginning of the softirq handler. When invoked frequently, this macro
 * will invoke rcu_softirq_qs() every 100 milliseconds thereafter, which will
 * provide both RCU and RCU-Tasks quiescent states. Note that this macro
 * modifies its old_ts argument.
 *
 * Because regions of code that have disabled softirq act as RCU read-side
 * critical sections, this macro should be invoked with softirq (and
 * preemption) enabled.
 *
 * The macro is not needed when CONFIG_PREEMPT_RT is defined. RT kernels would
 * have more chance to invoke schedule() calls and provide necessary quiescent
 * states. As a contrast, calling cond_resched() only won't achieve the same
 * effect because cond_resched() does not provide RCU-Tasks quiescent states.
 */
#define rcu_softirq_qs_periodic(old_ts) \
do { \
        if (!IS_ENABLED(CONFIG_PREEMPT_RT) && \
            time_after(jiffies, (old_ts) + HZ / 10)) { \
                preempt_disable(); \
                rcu_softirq_qs(); \
                preempt_enable(); \
                (old_ts) = jiffies; \
        } \
} while (0)

/*
 * Infrastructure to implement the synchronize_() primitives in
 * TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
 */

#if defined(CONFIG_TREE_RCU)
#include <linux/rcutree.h>
#elif defined(CONFIG_TINY_RCU)
#include <linux/rcutiny.h>
#else
#error "Unknown RCU implementation specified to kernel configuration"
#endif

/*
 * The init_rcu_head_on_stack() and destroy_rcu_head_on_stack() calls
 * are needed for dynamic initialization and destruction of rcu_head
 * on the stack, and init_rcu_head()/destroy_rcu_head() are needed for
 * dynamic initialization and destruction of statically allocated rcu_head
 * structures.  However, rcu_head structures allocated dynamically in the
 * heap don't need any initialization.
 */
#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
void init_rcu_head(struct rcu_head *head);
void destroy_rcu_head(struct rcu_head *head);
void init_rcu_head_on_stack(struct rcu_head *head);
void destroy_rcu_head_on_stack(struct rcu_head *head);
#else /* !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
static inline void init_rcu_head(struct rcu_head *head) { }
static inline void destroy_rcu_head(struct rcu_head *head) { }
static inline void init_rcu_head_on_stack(struct rcu_head *head) { }
static inline void destroy_rcu_head_on_stack(struct rcu_head *head) { }
#endif        /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */

#if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU)
bool rcu_lockdep_current_cpu_online(void);
#else /* #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
static inline bool rcu_lockdep_current_cpu_online(void) { return true; }
#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */

extern struct lockdep_map rcu_lock_map;
extern struct lockdep_map rcu_bh_lock_map;
extern struct lockdep_map rcu_sched_lock_map;
extern struct lockdep_map rcu_callback_map;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

static inline void rcu_lock_acquire(struct lockdep_map *map)
{
        lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
}

static inline void rcu_try_lock_acquire(struct lockdep_map *map)
{
        lock_acquire(map, 0, 1, 2, 0, NULL, _THIS_IP_);
}

static inline void rcu_lock_release(struct lockdep_map *map)
{
        lock_release(map, _THIS_IP_);
}

int debug_lockdep_rcu_enabled(void);
int rcu_read_lock_held(void);
int rcu_read_lock_bh_held(void);
int rcu_read_lock_sched_held(void);
int rcu_read_lock_any_held(void);

#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

# define rcu_lock_acquire(a)                do { } while (0)
# define rcu_try_lock_acquire(a)        do { } while (0)
# define rcu_lock_release(a)                do { } while (0)

static inline int rcu_read_lock_held(void)
{
        return 1;
}

static inline int rcu_read_lock_bh_held(void)
{
        return 1;
}

static inline int rcu_read_lock_sched_held(void)
{
        return !preemptible();
}

static inline int rcu_read_lock_any_held(void)
{
        return !preemptible();
}

static inline int debug_lockdep_rcu_enabled(void)
{
        return 0;
}

#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

#ifdef CONFIG_PROVE_RCU

/**
 * RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
 * @c: condition to check
 * @s: informative message
 *
 * This checks debug_lockdep_rcu_enabled() before checking (c) to
 * prevent early boot splats due to lockdep not yet being initialized,
 * and rechecks it after checking (c) to prevent false-positive splats
 * due to races with lockdep being disabled.  See commit 3066820034b5dd
 * ("rcu: Reject RCU_LOCKDEP_WARN() false positives") for more detail.
 */
#define RCU_LOCKDEP_WARN(c, s)                                                \
        do {                                                                \
                static bool __section(".data..unlikely") __warned;        \
                if (debug_lockdep_rcu_enabled() && (c) &&                \
                    debug_lockdep_rcu_enabled() && !__warned) {                \
                        __warned = true;                                \
                        lockdep_rcu_suspicious(__FILE__, __LINE__, s);        \
                }                                                        \
        } while (0)

#ifndef CONFIG_PREEMPT_RCU
static inline void rcu_preempt_sleep_check(void)
{
        RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
                         "Illegal context switch in RCU read-side critical section");
}
#else // #ifndef CONFIG_PREEMPT_RCU
static inline void rcu_preempt_sleep_check(void) { }
#endif // #else // #ifndef CONFIG_PREEMPT_RCU

#define rcu_sleep_check()                                                \
        do {                                                                \
                rcu_preempt_sleep_check();                                \
                if (!IS_ENABLED(CONFIG_PREEMPT_RT))                        \
                    RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map),        \
                                 "Illegal context switch in RCU-bh read-side critical section"); \
                RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map),        \
                                 "Illegal context switch in RCU-sched read-side critical section"); \
        } while (0)

// See RCU_LOCKDEP_WARN() for an explanation of the double call to
// debug_lockdep_rcu_enabled().
static inline bool lockdep_assert_rcu_helper(bool c)
{
        return debug_lockdep_rcu_enabled() &&
               (c || !rcu_is_watching() || !rcu_lockdep_current_cpu_online()) &&
               debug_lockdep_rcu_enabled();
}

/**
 * lockdep_assert_in_rcu_read_lock - WARN if not protected by rcu_read_lock()
 *
 * Splats if lockdep is enabled and there is no rcu_read_lock() in effect.
 */
#define lockdep_assert_in_rcu_read_lock() \
        WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map)))

/**
 * lockdep_assert_in_rcu_read_lock_bh - WARN if not protected by rcu_read_lock_bh()
 *
 * Splats if lockdep is enabled and there is no rcu_read_lock_bh() in effect.
 * Note that local_bh_disable() and friends do not suffice here, instead an
 * actual rcu_read_lock_bh() is required.
 */
#define lockdep_assert_in_rcu_read_lock_bh() \
        WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_bh_lock_map)))

/**
 * lockdep_assert_in_rcu_read_lock_sched - WARN if not protected by rcu_read_lock_sched()
 *
 * Splats if lockdep is enabled and there is no rcu_read_lock_sched()
 * in effect.  Note that preempt_disable() and friends do not suffice here,
 * instead an actual rcu_read_lock_sched() is required.
 */
#define lockdep_assert_in_rcu_read_lock_sched() \
        WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_sched_lock_map)))

/**
 * lockdep_assert_in_rcu_reader - WARN if not within some type of RCU reader
 *
 * Splats if lockdep is enabled and there is no RCU reader of any
 * type in effect.  Note that regions of code protected by things like
 * preempt_disable, local_bh_disable(), and local_irq_disable() all qualify
 * as RCU readers.
 *
 * Note that this will never trigger in PREEMPT_NONE or PREEMPT_VOLUNTARY
 * kernels that are not also built with PREEMPT_COUNT.  But if you have
 * lockdep enabled, you might as well also enable PREEMPT_COUNT.
 */
#define lockdep_assert_in_rcu_reader()                                                                \
        WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map) &&                        \
                                               !lock_is_held(&rcu_bh_lock_map) &&                \
                                               !lock_is_held(&rcu_sched_lock_map) &&                \
                                               preemptible()))

#else /* #ifdef CONFIG_PROVE_RCU */

#define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c))
#define rcu_sleep_check() do { } while (0)

#define lockdep_assert_in_rcu_read_lock() do { } while (0)
#define lockdep_assert_in_rcu_read_lock_bh() do { } while (0)
#define lockdep_assert_in_rcu_read_lock_sched() do { } while (0)
#define lockdep_assert_in_rcu_reader() do { } while (0)

#endif /* #else #ifdef CONFIG_PROVE_RCU */

/*
 * Helper functions for rcu_dereference_check(), rcu_dereference_protected()
 * and rcu_assign_pointer().  Some of these could be folded into their
 * callers, but they are left separate in order to ease introduction of
 * multiple pointers markings to match different RCU implementations
 * (e.g., __srcu), should this make sense in the future.
 */

#ifdef __CHECKER__
#define rcu_check_sparse(p, space) \
        ((void)(((typeof(*p) space *)p) == p))
#else /* #ifdef __CHECKER__ */
#define rcu_check_sparse(p, space)
#endif /* #else #ifdef __CHECKER__ */

#define __unrcu_pointer(p, local)                                        \
({                                                                        \
        typeof(*p) *local = (typeof(*p) *__force)(p);                        \
        rcu_check_sparse(p, __rcu);                                        \
        ((typeof(*p) __force __kernel *)(local));                         \
})
/**
 * unrcu_pointer - mark a pointer as not being RCU protected
 * @p: pointer needing to lose its __rcu property
 *
 * Converts @p from an __rcu pointer to a __kernel pointer.
 * This allows an __rcu pointer to be used with xchg() and friends.
 */
#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu))

#define __rcu_access_pointer(p, local, space) \
({ \
        typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
        rcu_check_sparse(p, space); \
        ((typeof(*p) __force __kernel *)(local)); \
})
#define __rcu_dereference_check(p, local, c, space) \
({ \
        /* Dependency order vs. p above. */ \
        typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
        RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
        rcu_check_sparse(p, space); \
        ((typeof(*p) __force __kernel *)(local)); \
})
#define __rcu_dereference_protected(p, local, c, space) \
({ \
        RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
        rcu_check_sparse(p, space); \
        ((typeof(*p) __force __kernel *)(p)); \
})
#define __rcu_dereference_raw(p, local) \
({ \
        /* Dependency order vs. p above. */ \
        typeof(p) local = READ_ONCE(p); \
        ((typeof(*p) __force __kernel *)(local)); \
})
#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu))

/**
 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
 * @v: The value to statically initialize with.
 */
#define RCU_INITIALIZER(v) (typeof(*(v)) __force __rcu *)(v)

/**
 * rcu_assign_pointer() - assign to RCU-protected pointer
 * @p: pointer to assign to
 * @v: value to assign (publish)
 *
 * Assigns the specified value to the specified RCU-protected
 * pointer, ensuring that any concurrent RCU readers will see
 * any prior initialization.
 *
 * Inserts memory barriers on architectures that require them
 * (which is most of them), and also prevents the compiler from
 * reordering the code that initializes the structure after the pointer
 * assignment.  More importantly, this call documents which pointers
 * will be dereferenced by RCU read-side code.
 *
 * In some special cases, you may use RCU_INIT_POINTER() instead
 * of rcu_assign_pointer().  RCU_INIT_POINTER() is a bit faster due
 * to the fact that it does not constrain either the CPU or the compiler.
 * That said, using RCU_INIT_POINTER() when you should have used
 * rcu_assign_pointer() is a very bad thing that results in
 * impossible-to-diagnose memory corruption.  So please be careful.
 * See the RCU_INIT_POINTER() comment header for details.
 *
 * Note that rcu_assign_pointer() evaluates each of its arguments only
 * once, appearances notwithstanding.  One of the "extra" evaluations
 * is in typeof() and the other visible only to sparse (__CHECKER__),
 * neither of which actually execute the argument.  As with most cpp
 * macros, this execute-arguments-only-once property is important, so
 * please be careful when making changes to rcu_assign_pointer() and the
 * other macros that it invokes.
 */
#define rcu_assign_pointer(p, v)                                              \
do {                                                                              \
        uintptr_t _r_a_p__v = (uintptr_t)(v);                                      \
        rcu_check_sparse(p, __rcu);                                              \
                                                                              \
        if (__builtin_constant_p(v) && (_r_a_p__v) == (uintptr_t)NULL)              \
                WRITE_ONCE((p), (typeof(p))(_r_a_p__v));                      \
        else                                                                      \
                smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \
} while (0)

/**
 * rcu_replace_pointer() - replace an RCU pointer, returning its old value
 * @rcu_ptr: RCU pointer, whose old value is returned
 * @ptr: regular pointer
 * @c: the lockdep conditions under which the dereference will take place
 *
 * Perform a replacement, where @rcu_ptr is an RCU-annotated
 * pointer and @c is the lockdep argument that is passed to the
 * rcu_dereference_protected() call used to read that pointer.  The old
 * value of @rcu_ptr is returned, and @rcu_ptr is set to @ptr.
 */
#define rcu_replace_pointer(rcu_ptr, ptr, c)                                \
({                                                                        \
        typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c));        \
        rcu_assign_pointer((rcu_ptr), (ptr));                                \
        __tmp;                                                                \
})

/**
 * rcu_access_pointer() - fetch RCU pointer with no dereferencing
 * @p: The pointer to read
 *
 * Return the value of the specified RCU-protected pointer, but omit the
 * lockdep checks for being in an RCU read-side critical section.  This is
 * useful when the value of this pointer is accessed, but the pointer is
 * not dereferenced, for example, when testing an RCU-protected pointer
 * against NULL.  Although rcu_access_pointer() may also be used in cases
 * where update-side locks prevent the value of the pointer from changing,
 * you should instead use rcu_dereference_protected() for this use case.
 * Within an RCU read-side critical section, there is little reason to
 * use rcu_access_pointer().
 *
 * It is usually best to test the rcu_access_pointer() return value
 * directly in order to avoid accidental dereferences being introduced
 * by later inattentive changes.  In other words, assigning the
 * rcu_access_pointer() return value to a local variable results in an
 * accident waiting to happen.
 *
 * It is also permissible to use rcu_access_pointer() when read-side
 * access to the pointer was removed at least one grace period ago, as is
 * the case in the context of the RCU callback that is freeing up the data,
 * or after a synchronize_rcu() returns.  This can be useful when tearing
 * down multi-linked structures after a grace period has elapsed.  However,
 * rcu_dereference_protected() is normally preferred for this use case.
 */
#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)

/**
 * rcu_dereference_check() - rcu_dereference with debug checking
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * Do an rcu_dereference(), but check that the conditions under which the
 * dereference will take place are correct.  Typically the conditions
 * indicate the various locking conditions that should be held at that
 * point.  The check should return true if the conditions are satisfied.
 * An implicit check for being in an RCU read-side critical section
 * (rcu_read_lock()) is included.
 *
 * For example:
 *
 *        bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock));
 *
 * could be used to indicate to lockdep that foo->bar may only be dereferenced
 * if either rcu_read_lock() is held, or that the lock required to replace
 * the bar struct at foo->bar is held.
 *
 * Note that the list of conditions may also include indications of when a lock
 * need not be held, for example during initialisation or destruction of the
 * target struct:
 *
 *        bar = rcu_dereference_check(foo->bar, lockdep_is_held(&foo->lock) ||
 *                                              atomic_read(&foo->usage) == 0);
 *
 * Inserts memory barriers on architectures that require them
 * (currently only the Alpha), prevents the compiler from refetching
 * (and from merging fetches), and, more importantly, documents exactly
 * which pointers are protected by RCU and checks that the pointer is
 * annotated as __rcu.
 */
#define rcu_dereference_check(p, c) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
                                (c) || rcu_read_lock_held(), __rcu)

/**
 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * This is the RCU-bh counterpart to rcu_dereference_check().  However,
 * please note that starting in v5.0 kernels, vanilla RCU grace periods
 * wait for local_bh_disable() regions of code in addition to regions of
 * code demarked by rcu_read_lock() and rcu_read_unlock().  This means
 * that synchronize_rcu(), call_rcu, and friends all take not only
 * rcu_read_lock() but also rcu_read_lock_bh() into account.
 */
#define rcu_dereference_bh_check(p, c) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
                                (c) || rcu_read_lock_bh_held(), __rcu)

/**
 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * This is the RCU-sched counterpart to rcu_dereference_check().
 * However, please note that starting in v5.0 kernels, vanilla RCU grace
 * periods wait for preempt_disable() regions of code in addition to
 * regions of code demarked by rcu_read_lock() and rcu_read_unlock().
 * This means that synchronize_rcu(), call_rcu, and friends all take not
 * only rcu_read_lock() but also rcu_read_lock_sched() into account.
 */
#define rcu_dereference_sched_check(p, c) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
                                (c) || rcu_read_lock_sched_held(), \
                                __rcu)

/*
 * The tracing infrastructure traces RCU (we want that), but unfortunately
 * some of the RCU checks causes tracing to lock up the system.
 *
 * The no-tracing version of rcu_dereference_raw() must not call
 * rcu_read_lock_held().
 */
#define rcu_dereference_raw_check(p) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu)

/**
 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
 * @p: The pointer to read, prior to dereferencing
 * @c: The conditions under which the dereference will take place
 *
 * Return the value of the specified RCU-protected pointer, but omit
 * the READ_ONCE().  This is useful in cases where update-side locks
 * prevent the value of the pointer from changing.  Please note that this
 * primitive does *not* prevent the compiler from repeating this reference
 * or combining it with other references, so it should not be used without
 * protection of appropriate locks.
 *
 * This function is only for update-side use.  Using this function
 * when protected only by rcu_read_lock() will result in infrequent
 * but very ugly failures.
 */
#define rcu_dereference_protected(p, c) \
        __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu)


/**
 * rcu_dereference() - fetch RCU-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
 *
 * This is a simple wrapper around rcu_dereference_check().
 */
#define rcu_dereference(p) rcu_dereference_check(p, 0)

/**
 * rcu_dereference_bh() - fetch an RCU-bh-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
 *
 * Makes rcu_dereference_check() do the dirty work.
 */
#define rcu_dereference_bh(p) rcu_dereference_bh_check(p, 0)

/**
 * rcu_dereference_sched() - fetch RCU-sched-protected pointer for dereferencing
 * @p: The pointer to read, prior to dereferencing
 *
 * Makes rcu_dereference_check() do the dirty work.
 */
#define rcu_dereference_sched(p) rcu_dereference_sched_check(p, 0)

/**
 * rcu_pointer_handoff() - Hand off a pointer from RCU to other mechanism
 * @p: The pointer to hand off
 *
 * This is simply an identity function, but it documents where a pointer
 * is handed off from RCU to some other synchronization mechanism, for
 * example, reference counting or locking.  In C11, it would map to
 * kill_dependency().  It could be used as follows::
 *
 *        rcu_read_lock();
 *        p = rcu_dereference(gp);
 *        long_lived = is_long_lived(p);
 *        if (long_lived) {
 *                if (!atomic_inc_not_zero(p->refcnt))
 *                        long_lived = false;
 *                else
 *                        p = rcu_pointer_handoff(p);
 *        }
 *        rcu_read_unlock();
 */
#define rcu_pointer_handoff(p) (p)

/**
 * rcu_read_lock() - mark the beginning of an RCU read-side critical section
 *
 * When synchronize_rcu() is invoked on one CPU while other CPUs
 * are within RCU read-side critical sections, then the
 * synchronize_rcu() is guaranteed to block until after all the other
 * CPUs exit their critical sections.  Similarly, if call_rcu() is invoked
 * on one CPU while other CPUs are within RCU read-side critical
 * sections, invocation of the corresponding RCU callback is deferred
 * until after the all the other CPUs exit their critical sections.
 *
 * Both synchronize_rcu() and call_rcu() also wait for regions of code
 * with preemption disabled, including regions of code with interrupts or
 * softirqs disabled.
 *
 * Note, however, that RCU callbacks are permitted to run concurrently
 * with new RCU read-side critical sections.  One way that this can happen
 * is via the following sequence of events: (1) CPU 0 enters an RCU
 * read-side critical section, (2) CPU 1 invokes call_rcu() to register
 * an RCU callback, (3) CPU 0 exits the RCU read-side critical section,
 * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU
 * callback is invoked.  This is legal, because the RCU read-side critical
 * section that was running concurrently with the call_rcu() (and which
 * therefore might be referencing something that the corresponding RCU
 * callback would free up) has completed before the corresponding
 * RCU callback is invoked.
 *
 * RCU read-side critical sections may be nested.  Any deferred actions
 * will be deferred until the outermost RCU read-side critical section
 * completes.
 *
 * You can avoid reading and understanding the next paragraph by
 * following this rule: don't put anything in an rcu_read_lock() RCU
 * read-side critical section that would block in a !PREEMPTION kernel.
 * But if you want the full story, read on!
 *
 * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
 * it is illegal to block while in an RCU read-side critical section.
 * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
 * kernel builds, RCU read-side critical sections may be preempted,
 * but explicit blocking is illegal.  Finally, in preemptible RCU
 * implementations in real-time (with -rt patchset) kernel builds, RCU
 * read-side critical sections may be preempted and they may also block, but
 * only when acquiring spinlocks that are subject to priority inheritance.
 */
static __always_inline void rcu_read_lock(void)
{
        __rcu_read_lock();
        __acquire(RCU);
        rcu_lock_acquire(&rcu_lock_map);
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_lock() used illegally while idle");
}

/*
 * So where is rcu_write_lock()?  It does not exist, as there is no
 * way for writers to lock out RCU readers.  This is a feature, not
 * a bug -- this property is what provides RCU's performance benefits.
 * Of course, writers must coordinate with each other.  The normal
 * spinlock primitives work well for this, but any other technique may be
 * used as well.  RCU does not care how the writers keep out of each
 * others' way, as long as they do so.
 */

/**
 * rcu_read_unlock() - marks the end of an RCU read-side critical section.
 *
 * In almost all situations, rcu_read_unlock() is immune from deadlock.
 * This deadlock immunity also extends to the scheduler's runqueue
 * and priority-inheritance spinlocks, courtesy of the quiescent-state
 * deferral that is carried out when rcu_read_unlock() is invoked with
 * interrupts disabled.
 *
 * See rcu_read_lock() for more information.
 */
static inline void rcu_read_unlock(void)
{
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_unlock() used illegally while idle");
        rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */
        __release(RCU);
        __rcu_read_unlock();
}

/**
 * rcu_read_lock_bh() - mark the beginning of an RCU-bh critical section
 *
 * This is equivalent to rcu_read_lock(), but also disables softirqs.
 * Note that anything else that disables softirqs can also serve as an RCU
 * read-side critical section.  However, please note that this equivalence
 * applies only to v5.0 and later.  Before v5.0, rcu_read_lock() and
 * rcu_read_lock_bh() were unrelated.
 *
 * Note that rcu_read_lock_bh() and the matching rcu_read_unlock_bh()
 * must occur in the same context, for example, it is illegal to invoke
 * rcu_read_unlock_bh() from one task if the matching rcu_read_lock_bh()
 * was invoked from some other task.
 */
static inline void rcu_read_lock_bh(void)
{
        local_bh_disable();
        __acquire(RCU_BH);
        rcu_lock_acquire(&rcu_bh_lock_map);
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_lock_bh() used illegally while idle");
}

/**
 * rcu_read_unlock_bh() - marks the end of a softirq-only RCU critical section
 *
 * See rcu_read_lock_bh() for more information.
 */
static inline void rcu_read_unlock_bh(void)
{
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_unlock_bh() used illegally while idle");
        rcu_lock_release(&rcu_bh_lock_map);
        __release(RCU_BH);
        local_bh_enable();
}

/**
 * rcu_read_lock_sched() - mark the beginning of a RCU-sched critical section
 *
 * This is equivalent to rcu_read_lock(), but also disables preemption.
 * Read-side critical sections can also be introduced by anything else that
 * disables preemption, including local_irq_disable() and friends.  However,
 * please note that the equivalence to rcu_read_lock() applies only to
 * v5.0 and later.  Before v5.0, rcu_read_lock() and rcu_read_lock_sched()
 * were unrelated.
 *
 * Note that rcu_read_lock_sched() and the matching rcu_read_unlock_sched()
 * must occur in the same context, for example, it is illegal to invoke
 * rcu_read_unlock_sched() from process context if the matching
 * rcu_read_lock_sched() was invoked from an NMI handler.
 */
static inline void rcu_read_lock_sched(void)
{
        preempt_disable();
        __acquire(RCU_SCHED);
        rcu_lock_acquire(&rcu_sched_lock_map);
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_lock_sched() used illegally while idle");
}

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_lock_sched_notrace(void)
{
        preempt_disable_notrace();
        __acquire(RCU_SCHED);
}

/**
 * rcu_read_unlock_sched() - marks the end of a RCU-classic critical section
 *
 * See rcu_read_lock_sched() for more information.
 */
static inline void rcu_read_unlock_sched(void)
{
        RCU_LOCKDEP_WARN(!rcu_is_watching(),
                         "rcu_read_unlock_sched() used illegally while idle");
        rcu_lock_release(&rcu_sched_lock_map);
        __release(RCU_SCHED);
        preempt_enable();
}

/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
static inline notrace void rcu_read_unlock_sched_notrace(void)
{
        __release(RCU_SCHED);
        preempt_enable_notrace();
}

/**
 * RCU_INIT_POINTER() - initialize an RCU protected pointer
 * @p: The pointer to be initialized.
 * @v: The value to initialized the pointer to.
 *
 * Initialize an RCU-protected pointer in special cases where readers
 * do not need ordering constraints on the CPU or the compiler.  These
 * special cases are:
 *
 * 1.        This use of RCU_INIT_POINTER() is NULLing out the pointer *or*
 * 2.        The caller has taken whatever steps are required to prevent
 *        RCU readers from concurrently accessing this pointer *or*
 * 3.        The referenced data structure has already been exposed to
 *        readers either at compile time or via rcu_assign_pointer() *and*
 *
 *        a.        You have not made *any* reader-visible changes to
 *                this structure since then *or*
 *        b.        It is OK for readers accessing this structure from its
 *                new location to see the old state of the structure.  (For
 *                example, the changes were to statistical counters or to
 *                other state where exact synchronization is not required.)
 *
 * Failure to follow these rules governing use of RCU_INIT_POINTER() will
 * result in impossible-to-diagnose memory corruption.  As in the structures
 * will look OK in crash dumps, but any concurrent RCU readers might
 * see pre-initialized values of the referenced data structure.  So
 * please be very careful how you use RCU_INIT_POINTER()!!!
 *
 * If you are creating an RCU-protected linked structure that is accessed
 * by a single external-to-structure RCU-protected pointer, then you may
 * use RCU_INIT_POINTER() to initialize the internal RCU-protected
 * pointers, but you must use rcu_assign_pointer() to initialize the
 * external-to-structure pointer *after* you have completely initialized
 * the reader-accessible portions of the linked structure.
 *
 * Note that unlike rcu_assign_pointer(), RCU_INIT_POINTER() provides no
 * ordering guarantees for either the CPU or the compiler.
 */
#define RCU_INIT_POINTER(p, v) \
        do { \
                rcu_check_sparse(p, __rcu); \
                WRITE_ONCE(p, RCU_INITIALIZER(v)); \
        } while (0)

/**
 * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer
 * @p: The pointer to be initialized.
 * @v: The value to initialized the pointer to.
 *
 * GCC-style initialization for an RCU-protected pointer in a structure field.
 */
#define RCU_POINTER_INITIALIZER(p, v) \
                .p = RCU_INITIALIZER(v)

/**
 * kfree_rcu() - kfree an object after a grace period.
 * @ptr: pointer to kfree for double-argument invocations.
 * @rhf: the name of the struct rcu_head within the type of @ptr.
 *
 * Many rcu callbacks functions just call kfree() on the base structure.
 * These functions are trivial, but their size adds up, and furthermore
 * when they are used in a kernel module, that module must invoke the
 * high-latency rcu_barrier() function at module-unload time.
 *
 * The kfree_rcu() function handles this issue. In order to have a universal
 * callback function handling different offsets of rcu_head, the callback needs
 * to determine the starting address of the freed object, which can be a large
 * kmalloc or vmalloc allocation. To allow simply aligning the pointer down to
 * page boundary for those, only offsets up to 4095 bytes can be accommodated.
 * If the offset is larger than 4095 bytes, a compile-time error will
 * be generated in kvfree_rcu_arg_2(). If this error is triggered, you can
 * either fall back to use of call_rcu() or rearrange the structure to
 * position the rcu_head structure into the first 4096 bytes.
 *
 * The object to be freed can be allocated either by kmalloc() or
 * kmem_cache_alloc().
 *
 * Note that the allowable offset might decrease in the future.
 *
 * The BUILD_BUG_ON check must not involve any function calls, hence the
 * checks are done in macros here.
 */
#define kfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf)
#define kvfree_rcu(ptr, rhf) kvfree_rcu_arg_2(ptr, rhf)

/**
 * kfree_rcu_mightsleep() - kfree an object after a grace period.
 * @ptr: pointer to kfree for single-argument invocations.
 *
 * When it comes to head-less variant, only one argument
 * is passed and that is just a pointer which has to be
 * freed after a grace period. Therefore the semantic is
 *
 *     kfree_rcu_mightsleep(ptr);
 *
 * where @ptr is the pointer to be freed by kvfree().
 *
 * Please note, head-less way of freeing is permitted to
 * use from a context that has to follow might_sleep()
 * annotation. Otherwise, please switch and embed the
 * rcu_head structure within the type of @ptr.
 */
#define kfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr)
#define kvfree_rcu_mightsleep(ptr) kvfree_rcu_arg_1(ptr)

/*
 * In mm/slab_common.c, no suitable header to include here.
 */
void kvfree_call_rcu(struct rcu_head *head, void *ptr);

/*
 * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the
 * comment of kfree_rcu() for details.
 */
#define kvfree_rcu_arg_2(ptr, rhf)                                        \
do {                                                                        \
        typeof (ptr) ___p = (ptr);                                        \
                                                                        \
        if (___p) {                                                        \
                BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096);        \
                kvfree_call_rcu(&((___p)->rhf), (void *) (___p));        \
        }                                                                \
} while (0)

#define kvfree_rcu_arg_1(ptr)                                        \
do {                                                                \
        typeof(ptr) ___p = (ptr);                                \
                                                                \
        if (___p)                                                \
                kvfree_call_rcu(NULL, (void *) (___p));                \
} while (0)

/*
 * Place this after a lock-acquisition primitive to guarantee that
 * an UNLOCK+LOCK pair acts as a full barrier.  This guarantee applies
 * if the UNLOCK and LOCK are executed by the same CPU or if the
 * UNLOCK and LOCK operate on the same lock variable.
 */
#ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE
#define smp_mb__after_unlock_lock()        smp_mb()  /* Full ordering for lock. */
#else /* #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */
#define smp_mb__after_unlock_lock()        do { } while (0)
#endif /* #else #ifdef CONFIG_ARCH_WEAK_RELEASE_ACQUIRE */


/* Has the specified rcu_head structure been handed to call_rcu()? */

/**
 * rcu_head_init - Initialize rcu_head for rcu_head_after_call_rcu()
 * @rhp: The rcu_head structure to initialize.
 *
 * If you intend to invoke rcu_head_after_call_rcu() to test whether a
 * given rcu_head structure has already been passed to call_rcu(), then
 * you must also invoke this rcu_head_init() function on it just after
 * allocating that structure.  Calls to this function must not race with
 * calls to call_rcu(), rcu_head_after_call_rcu(), or callback invocation.
 */
static inline void rcu_head_init(struct rcu_head *rhp)
{
        rhp->func = (rcu_callback_t)~0L;
}

/**
 * rcu_head_after_call_rcu() - Has this rcu_head been passed to call_rcu()?
 * @rhp: The rcu_head structure to test.
 * @f: The function passed to call_rcu() along with @rhp.
 *
 * Returns @true if the @rhp has been passed to call_rcu() with @func,
 * and @false otherwise.  Emits a warning in any other case, including
 * the case where @rhp has already been invoked after a grace period.
 * Calls to this function must not race with callback invocation.  One way
 * to avoid such races is to enclose the call to rcu_head_after_call_rcu()
 * in an RCU read-side critical section that includes a read-side fetch
 * of the pointer to the structure containing @rhp.
 */
static inline bool
rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f)
{
        rcu_callback_t func = READ_ONCE(rhp->func);

        if (func == f)
                return true;
        WARN_ON_ONCE(func != (rcu_callback_t)~0L);
        return false;
}

/* kernel/ksysfs.c definitions */
extern int rcu_expedited;
extern int rcu_normal;

DEFINE_LOCK_GUARD_0(rcu,
        do {
                rcu_read_lock();
                /*
                 * sparse doesn't call the cleanup function,
                 * so just release immediately and don't track
                 * the context. We don't need to anyway, since
                 * the whole point of the guard is to not need
                 * the explicit unlock.
                 */
                __release(RCU);
        } while (0),
        rcu_read_unlock())

#endif /* __LINUX_RCUPDATE_H */







































































































































































































































































































































































































































































































































    2 























    6 


    4 




    2 
















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * xfrm_device.c - IPsec device offloading code.
 *
 * Copyright (c) 2015 secunet Security Networks AG
 *
 * Author:
 * Steffen Klassert <steffen.klassert@secunet.com>
 */

#include <linux/errno.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <net/dst.h>
#include <net/gso.h>
#include <net/xfrm.h>
#include <linux/notifier.h>

#ifdef CONFIG_XFRM_OFFLOAD
static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
                                  unsigned int hsize)
{
        struct xfrm_offload *xo = xfrm_offload(skb);

        skb_reset_mac_len(skb);
        if (xo->flags & XFRM_GSO_SEGMENT)
                skb->transport_header -= x->props.header_len;

        pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
}

static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
                                    unsigned int hsize)

{
        struct xfrm_offload *xo = xfrm_offload(skb);

        if (xo->flags & XFRM_GSO_SEGMENT)
                skb->transport_header = skb->network_header + hsize;

        skb_reset_mac_len(skb);
        pskb_pull(skb,
                  skb->mac_len + x->props.header_len - x->props.enc_hdr_len);
}

static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
                                  unsigned int hsize)
{
        struct xfrm_offload *xo = xfrm_offload(skb);
        int phlen = 0;

        if (xo->flags & XFRM_GSO_SEGMENT)
                skb->transport_header = skb->network_header + hsize;

        skb_reset_mac_len(skb);
        if (x->sel.family != AF_INET6) {
                phlen = IPV4_BEET_PHMAXLEN;
                if (x->outer_mode.family == AF_INET6)
                        phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
        }

        pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
}

/* Adjust pointers into the packet when IPsec is done at layer2 */
static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
{
        switch (x->outer_mode.encap) {
        case XFRM_MODE_IPTFS:
        case XFRM_MODE_TUNNEL:
                if (x->outer_mode.family == AF_INET)
                        return __xfrm_mode_tunnel_prep(x, skb,
                                                       sizeof(struct iphdr));
                if (x->outer_mode.family == AF_INET6)
                        return __xfrm_mode_tunnel_prep(x, skb,
                                                       sizeof(struct ipv6hdr));
                break;
        case XFRM_MODE_TRANSPORT:
                if (x->outer_mode.family == AF_INET)
                        return __xfrm_transport_prep(x, skb,
                                                     sizeof(struct iphdr));
                if (x->outer_mode.family == AF_INET6)
                        return __xfrm_transport_prep(x, skb,
                                                     sizeof(struct ipv6hdr));
                break;
        case XFRM_MODE_BEET:
                if (x->outer_mode.family == AF_INET)
                        return __xfrm_mode_beet_prep(x, skb,
                                                     sizeof(struct iphdr));
                if (x->outer_mode.family == AF_INET6)
                        return __xfrm_mode_beet_prep(x, skb,
                                                     sizeof(struct ipv6hdr));
                break;
        case XFRM_MODE_ROUTEOPTIMIZATION:
        case XFRM_MODE_IN_TRIGGER:
                break;
        }
}

static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb)
{
        struct xfrm_offload *xo = xfrm_offload(skb);
        __u32 seq = xo->seq.low;

        seq += skb_shinfo(skb)->gso_segs;
        if (unlikely(seq < xo->seq.low))
                return true;

        return false;
}

struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
{
        int err;
        unsigned long flags;
        struct xfrm_state *x;
        struct softnet_data *sd;
        struct sk_buff *skb2, *nskb, *pskb = NULL;
        netdev_features_t esp_features = features;
        struct xfrm_offload *xo = xfrm_offload(skb);
        struct net_device *dev = skb->dev;
        struct sec_path *sp;

        if (!xo || (xo->flags & XFRM_XMIT))
                return skb;

        if (!(features & NETIF_F_HW_ESP))
                esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);

        sp = skb_sec_path(skb);
        x = sp->xvec[sp->len - 1];
        if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
                return skb;

        /* The packet was sent to HW IPsec packet offload engine,
         * but to wrong device. Drop the packet, so it won't skip
         * XFRM stack.
         */
        if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) {
                kfree_skb(skb);
                dev_core_stats_tx_dropped_inc(dev);
                return NULL;
        }

        local_irq_save(flags);
        sd = this_cpu_ptr(&softnet_data);
        err = !skb_queue_empty(&sd->xfrm_backlog);
        local_irq_restore(flags);

        if (err) {
                *again = true;
                return skb;
        }

        if (skb_is_gso(skb) && unlikely(xmit_xfrm_check_overflow(skb))) {
                struct sk_buff *segs;

                /* Packet got rerouted, fixup features and segment it. */
                esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);

                segs = skb_gso_segment(skb, esp_features);
                if (IS_ERR(segs)) {
                        kfree_skb(skb);
                        dev_core_stats_tx_dropped_inc(dev);
                        return NULL;
                } else {
                        consume_skb(skb);
                        skb = segs;
                }
        }

        if (!skb->next) {
                esp_features |= skb->dev->gso_partial_features;
                xfrm_outer_mode_prep(x, skb);

                xo->flags |= XFRM_DEV_RESUME;

                err = x->type_offload->xmit(x, skb, esp_features);
                if (err) {
                        if (err == -EINPROGRESS)
                                return NULL;

                        XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
                        kfree_skb(skb);
                        return NULL;
                }

                skb_push(skb, skb->data - skb_mac_header(skb));

                return skb;
        }

        skb_list_walk_safe(skb, skb2, nskb) {
                esp_features |= skb->dev->gso_partial_features;
                skb_mark_not_on_list(skb2);

                xo = xfrm_offload(skb2);
                xo->flags |= XFRM_DEV_RESUME;

                xfrm_outer_mode_prep(x, skb2);

                err = x->type_offload->xmit(x, skb2, esp_features);
                if (!err) {
                        skb2->next = nskb;
                } else if (err != -EINPROGRESS) {
                        XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
                        skb2->next = nskb;
                        kfree_skb_list(skb2);
                        return NULL;
                } else {
                        if (skb == skb2)
                                skb = nskb;
                        else
                                pskb->next = nskb;

                        continue;
                }

                skb_push(skb2, skb2->data - skb_mac_header(skb2));
                pskb = skb2;
        }

        return skb;
}
EXPORT_SYMBOL_GPL(validate_xmit_xfrm);

int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
                       struct xfrm_user_offload *xuo,
                       struct netlink_ext_ack *extack)
{
        int err;
        struct dst_entry *dst;
        struct net_device *dev;
        struct xfrm_dev_offload *xso = &x->xso;
        xfrm_address_t *saddr;
        xfrm_address_t *daddr;
        bool is_packet_offload;

        if (xuo->flags &
            ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) {
                NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
                return -EINVAL;
        }

        if ((xuo->flags & XFRM_OFFLOAD_INBOUND && x->dir == XFRM_SA_DIR_OUT) ||
            (!(xuo->flags & XFRM_OFFLOAD_INBOUND) && x->dir == XFRM_SA_DIR_IN)) {
                NL_SET_ERR_MSG(extack, "Mismatched SA and offload direction");
                return -EINVAL;
        }

        if (xuo->flags & XFRM_OFFLOAD_INBOUND && x->if_id) {
                NL_SET_ERR_MSG(extack, "XFRM if_id is not supported in RX path");
                return -EINVAL;
        }

        is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;

        /* We don't yet support TFC padding. */
        if (x->tfcpad) {
                NL_SET_ERR_MSG(extack, "TFC padding can't be offloaded");
                return -EINVAL;
        }

        dev = dev_get_by_index(net, xuo->ifindex);
        if (!dev) {
                struct xfrm_dst_lookup_params params;

                if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
                        saddr = &x->props.saddr;
                        daddr = &x->id.daddr;
                } else {
                        saddr = &x->id.daddr;
                        daddr = &x->props.saddr;
                }

                memset(&params, 0, sizeof(params));
                params.net = net;
                params.saddr = saddr;
                params.daddr = daddr;
                params.mark = xfrm_smark_get(0, x);
                dst = __xfrm_dst_lookup(x->props.family, &params);
                if (IS_ERR(dst))
                        return (is_packet_offload) ? -EINVAL : 0;

                dev = dst->dev;

                dev_hold(dev);
                dst_release(dst);
        }

        if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
                xso->dev = NULL;
                dev_put(dev);
                return (is_packet_offload) ? -EINVAL : 0;
        }

        if (!is_packet_offload && x->props.flags & XFRM_STATE_ESN &&
            !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
                NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
                xso->dev = NULL;
                dev_put(dev);
                return -EINVAL;
        }

        xfrm_set_type_offload(x);
        if (!x->type_offload) {
                NL_SET_ERR_MSG(extack, "Type doesn't support offload");
                dev_put(dev);
                return -EINVAL;
        }

        xso->dev = dev;
        netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);

        if (xuo->flags & XFRM_OFFLOAD_INBOUND)
                xso->dir = XFRM_DEV_OFFLOAD_IN;
        else
                xso->dir = XFRM_DEV_OFFLOAD_OUT;

        if (is_packet_offload)
                xso->type = XFRM_DEV_OFFLOAD_PACKET;
        else
                xso->type = XFRM_DEV_OFFLOAD_CRYPTO;

        err = dev->xfrmdev_ops->xdo_dev_state_add(dev, x, extack);
        if (err) {
                xso->dev = NULL;
                xso->dir = 0;
                netdev_put(dev, &xso->dev_tracker);
                xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;

                xfrm_unset_type_offload(x);
                /* User explicitly requested packet offload mode and configured
                 * policy in addition to the XFRM state. So be civil to users,
                 * and return an error instead of taking fallback path.
                 */
                if ((err != -EOPNOTSUPP && !is_packet_offload) || is_packet_offload) {
                        NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this state");
                        return err;
                }
        }

        return 0;
}
EXPORT_SYMBOL_GPL(xfrm_dev_state_add);

int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
                        struct xfrm_user_offload *xuo, u8 dir,
                        struct netlink_ext_ack *extack)
{
        struct xfrm_dev_offload *xdo = &xp->xdo;
        struct net_device *dev;
        int err;

        if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) {
                /* We support only packet offload mode and it means
                 * that user must set XFRM_OFFLOAD_PACKET bit.
                 */
                NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
                return -EINVAL;
        }

        dev = dev_get_by_index(net, xuo->ifindex);
        if (!dev)
                return -EINVAL;

        if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) {
                xdo->dev = NULL;
                dev_put(dev);
                NL_SET_ERR_MSG(extack, "Policy offload is not supported");
                return -EINVAL;
        }

        xdo->dev = dev;
        netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
        xdo->type = XFRM_DEV_OFFLOAD_PACKET;
        switch (dir) {
        case XFRM_POLICY_IN:
                xdo->dir = XFRM_DEV_OFFLOAD_IN;
                break;
        case XFRM_POLICY_OUT:
                xdo->dir = XFRM_DEV_OFFLOAD_OUT;
                break;
        case XFRM_POLICY_FWD:
                xdo->dir = XFRM_DEV_OFFLOAD_FWD;
                break;
        default:
                xdo->dev = NULL;
                netdev_put(dev, &xdo->dev_tracker);
                NL_SET_ERR_MSG(extack, "Unrecognized offload direction");
                return -EINVAL;
        }

        err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
        if (err) {
                xdo->dev = NULL;
                xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
                xdo->dir = 0;
                netdev_put(dev, &xdo->dev_tracker);
                NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this policy");
                return err;
        }

        return 0;
}
EXPORT_SYMBOL_GPL(xfrm_dev_policy_add);

bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
        int mtu;
        struct dst_entry *dst = skb_dst(skb);
        struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
        struct net_device *dev = x->xso.dev;
        bool check_tunnel_size;

        if (x->xso.type == XFRM_DEV_OFFLOAD_UNSPECIFIED)
                return false;

        if ((dev == xfrm_dst_path(dst)->dev) && !xdst->child->xfrm) {
                mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
                if (skb->len <= mtu)
                        goto ok;

                if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
                        goto ok;
        }

        return false;

ok:
        check_tunnel_size = x->xso.type == XFRM_DEV_OFFLOAD_PACKET &&
                            x->props.mode == XFRM_MODE_TUNNEL;
        switch (x->props.family) {
        case AF_INET:
                /* Check for IPv4 options */
                if (ip_hdr(skb)->ihl != 5)
                        return false;
                if (check_tunnel_size && xfrm4_tunnel_check_size(skb))
                        return false;
                break;
        case AF_INET6:
                /* Check for IPv6 extensions */
                if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
                        return false;
                if (check_tunnel_size && xfrm6_tunnel_check_size(skb))
                        return false;
                break;
        default:
                break;
        }

        if (dev->xfrmdev_ops->xdo_dev_offload_ok)
                return dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);

        return true;
}
EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);

void xfrm_dev_resume(struct sk_buff *skb)
{
        struct net_device *dev = skb->dev;
        int ret = NETDEV_TX_BUSY;
        struct netdev_queue *txq;
        struct softnet_data *sd;
        unsigned long flags;

        rcu_read_lock();
        txq = netdev_core_pick_tx(dev, skb, NULL);

        HARD_TX_LOCK(dev, txq, smp_processor_id());
        if (!netif_xmit_frozen_or_stopped(txq))
                skb = dev_hard_start_xmit(skb, dev, txq, &ret);
        HARD_TX_UNLOCK(dev, txq);

        if (!dev_xmit_complete(ret)) {
                local_irq_save(flags);
                sd = this_cpu_ptr(&softnet_data);
                skb_queue_tail(&sd->xfrm_backlog, skb);
                raise_softirq_irqoff(NET_TX_SOFTIRQ);
                local_irq_restore(flags);
        }
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(xfrm_dev_resume);

void xfrm_dev_backlog(struct softnet_data *sd)
{
        struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
        struct sk_buff_head list;
        struct sk_buff *skb;

        if (skb_queue_empty(xfrm_backlog))
                return;

        __skb_queue_head_init(&list);

        spin_lock(&xfrm_backlog->lock);
        skb_queue_splice_init(xfrm_backlog, &list);
        spin_unlock(&xfrm_backlog->lock);

        while (!skb_queue_empty(&list)) {
                skb = __skb_dequeue(&list);
                xfrm_dev_resume(skb);
        }

}
#endif

static int xfrm_api_check(struct net_device *dev)
{
#ifdef CONFIG_XFRM_OFFLOAD
        if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
            !(dev->features & NETIF_F_HW_ESP))
                return NOTIFY_BAD;

        if ((dev->features & NETIF_F_HW_ESP) &&
            (!(dev->xfrmdev_ops &&
               dev->xfrmdev_ops->xdo_dev_state_add &&
               dev->xfrmdev_ops->xdo_dev_state_delete)))
                return NOTIFY_BAD;
#else
        if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
                return NOTIFY_BAD;
#endif

        return NOTIFY_DONE;
}

static int xfrm_dev_down(struct net_device *dev)
{
        if (dev->features & NETIF_F_HW_ESP) {
                xfrm_dev_state_flush(dev_net(dev), dev, true);
                xfrm_dev_policy_flush(dev_net(dev), dev, true);
        }

        return NOTIFY_DONE;
}

static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);

        switch (event) {
        case NETDEV_REGISTER:
                return xfrm_api_check(dev);

        case NETDEV_FEAT_CHANGE:
                return xfrm_api_check(dev);

        case NETDEV_DOWN:
        case NETDEV_UNREGISTER:
                return xfrm_dev_down(dev);
        }
        return NOTIFY_DONE;
}

static struct notifier_block xfrm_dev_notifier = {
        .notifier_call        = xfrm_dev_event,
};

void __init xfrm_dev_init(void)
{
        register_netdevice_notifier(&xfrm_dev_notifier);
}


















































   37 













































   37 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM signal

#if !defined(_TRACE_SIGNAL_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SIGNAL_H

#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/tracepoint.h>

#define TP_STORE_SIGINFO(__entry, info)                                \
        do {                                                        \
                if (info == SEND_SIG_NOINFO) {                        \
                        __entry->errno        = 0;                        \
                        __entry->code        = SI_USER;                \
                } else if (info == SEND_SIG_PRIV) {                \
                        __entry->errno        = 0;                        \
                        __entry->code        = SI_KERNEL;                \
                } else {                                        \
                        __entry->errno        = info->si_errno;        \
                        __entry->code        = info->si_code;        \
                }                                                \
        } while (0)

#ifndef TRACE_HEADER_MULTI_READ
enum {
        TRACE_SIGNAL_DELIVERED,
        TRACE_SIGNAL_IGNORED,
        TRACE_SIGNAL_ALREADY_PENDING,
        TRACE_SIGNAL_OVERFLOW_FAIL,
        TRACE_SIGNAL_LOSE_INFO,
};
#endif

/**
 * signal_generate - called when a signal is generated
 * @sig: signal number
 * @info: pointer to struct siginfo
 * @task: pointer to struct task_struct
 * @group: shared or private
 * @result: TRACE_SIGNAL_*
 *
 * Current process sends a 'sig' signal to 'task' process with
 * 'info' siginfo. If 'info' is SEND_SIG_NOINFO or SEND_SIG_PRIV,
 * 'info' is not a pointer and you can't access its field. Instead,
 * SEND_SIG_NOINFO means that si_code is SI_USER, and SEND_SIG_PRIV
 * means that si_code is SI_KERNEL.
 */
TRACE_EVENT(signal_generate,

        TP_PROTO(int sig, struct kernel_siginfo *info, struct task_struct *task,
                        int group, int result),

        TP_ARGS(sig, info, task, group, result),

        TP_STRUCT__entry(
                __field(        int,        sig                        )
                __field(        int,        errno                        )
                __field(        int,        code                        )
                __array(        char,        comm,        TASK_COMM_LEN        )
                __field(        pid_t,        pid                        )
                __field(        int,        group                        )
                __field(        int,        result                        )
        ),

        TP_fast_assign(
                __entry->sig        = sig;
                TP_STORE_SIGINFO(__entry, info);
                memcpy(__entry->comm, task->comm, TASK_COMM_LEN);
                __entry->pid        = task->pid;
                __entry->group        = group;
                __entry->result        = result;
        ),

        TP_printk("sig=%d errno=%d code=%d comm=%s pid=%d grp=%d res=%d",
                  __entry->sig, __entry->errno, __entry->code,
                  __entry->comm, __entry->pid, __entry->group,
                  __entry->result)
);

/**
 * signal_deliver - called when a signal is delivered
 * @sig: signal number
 * @info: pointer to struct siginfo
 * @ka: pointer to struct k_sigaction
 *
 * A 'sig' signal is delivered to current process with 'info' siginfo,
 * and it will be handled by 'ka'. ka->sa.sa_handler can be SIG_IGN or
 * SIG_DFL.
 * Note that some signals reported by signal_generate tracepoint can be
 * lost, ignored or modified (by debugger) before hitting this tracepoint.
 * This means, this can show which signals are actually delivered, but
 * matching generated signals and delivered signals may not be correct.
 */
TRACE_EVENT(signal_deliver,

        TP_PROTO(int sig, struct kernel_siginfo *info, struct k_sigaction *ka),

        TP_ARGS(sig, info, ka),

        TP_STRUCT__entry(
                __field(        int,                sig                )
                __field(        int,                errno                )
                __field(        int,                code                )
                __field(        unsigned long,        sa_handler        )
                __field(        unsigned long,        sa_flags        )
        ),

        TP_fast_assign(
                __entry->sig        = sig;
                TP_STORE_SIGINFO(__entry, info);
                __entry->sa_handler        = (unsigned long)ka->sa.sa_handler;
                __entry->sa_flags        = ka->sa.sa_flags;
        ),

        TP_printk("sig=%d errno=%d code=%d sa_handler=%lx sa_flags=%lx",
                  __entry->sig, __entry->errno, __entry->code,
                  __entry->sa_handler, __entry->sa_flags)
);

#endif /* _TRACE_SIGNAL_H */

/* This part must be outside protection */
#include <trace/define_trace.h>
























































  136 


  136 













  136 





  136 



















































































































 1122 











 1125 
 1122 




 1127 











 1123 




  124 

 1123 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
// SPDX-License-Identifier: GPL-2.0
#include <linux/irq_work.h>
#include <linux/spinlock.h>
#include <linux/task_work.h>
#include <linux/resume_user_mode.h>

static struct callback_head work_exited; /* all we need is ->next == NULL */

#ifdef CONFIG_IRQ_WORK
static void task_work_set_notify_irq(struct irq_work *entry)
{
        test_and_set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
}
static DEFINE_PER_CPU(struct irq_work, irq_work_NMI_resume) =
        IRQ_WORK_INIT_HARD(task_work_set_notify_irq);
#endif

/**
 * task_work_add - ask the @task to execute @work->func()
 * @task: the task which should run the callback
 * @work: the callback to run
 * @notify: how to notify the targeted task
 *
 * Queue @work for task_work_run() below and notify the @task if @notify
 * is @TWA_RESUME, @TWA_SIGNAL, @TWA_SIGNAL_NO_IPI or @TWA_NMI_CURRENT.
 *
 * @TWA_SIGNAL works like signals, in that the it will interrupt the targeted
 * task and run the task_work, regardless of whether the task is currently
 * running in the kernel or userspace.
 * @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a
 * reschedule IPI to force the targeted task to reschedule and run task_work.
 * This can be advantageous if there's no strict requirement that the
 * task_work be run as soon as possible, just whenever the task enters the
 * kernel anyway.
 * @TWA_RESUME work is run only when the task exits the kernel and returns to
 * user mode, or before entering guest mode.
 * @TWA_NMI_CURRENT works like @TWA_RESUME, except it can only be used for the
 * current @task and if the current context is NMI.
 *
 * Fails if the @task is exiting/exited and thus it can't process this @work.
 * Otherwise @work->func() will be called when the @task goes through one of
 * the aforementioned transitions, or exits.
 *
 * If the targeted task is exiting, then an error is returned and the work item
 * is not queued. It's up to the caller to arrange for an alternative mechanism
 * in that case.
 *
 * Note: there is no ordering guarantee on works queued here. The task_work
 * list is LIFO.
 *
 * RETURNS:
 * 0 if succeeds or -ESRCH.
 */
int task_work_add(struct task_struct *task, struct callback_head *work,
                  enum task_work_notify_mode notify)
{
        struct callback_head *head;

        if (notify == TWA_NMI_CURRENT) {
                if (WARN_ON_ONCE(task != current))
                        return -EINVAL;
                if (!IS_ENABLED(CONFIG_IRQ_WORK))
                        return -EINVAL;
        } else {
                kasan_record_aux_stack(work);
        }

        head = READ_ONCE(task->task_works);
        do {
                if (unlikely(head == &work_exited))
                        return -ESRCH;
                work->next = head;
        } while (!try_cmpxchg(&task->task_works, &head, work));

        switch (notify) {
        case TWA_NONE:
                break;
        case TWA_RESUME:
                set_notify_resume(task);
                break;
        case TWA_SIGNAL:
                set_notify_signal(task);
                break;
        case TWA_SIGNAL_NO_IPI:
                __set_notify_signal(task);
                break;
#ifdef CONFIG_IRQ_WORK
        case TWA_NMI_CURRENT:
                irq_work_queue(this_cpu_ptr(&irq_work_NMI_resume));
                break;
#endif
        default:
                WARN_ON_ONCE(1);
                break;
        }

        return 0;
}

/**
 * task_work_cancel_match - cancel a pending work added by task_work_add()
 * @task: the task which should execute the work
 * @match: match function to call
 * @data: data to be passed in to match function
 *
 * RETURNS:
 * The found work or NULL if not found.
 */
struct callback_head *
task_work_cancel_match(struct task_struct *task,
                       bool (*match)(struct callback_head *, void *data),
                       void *data)
{
        struct callback_head **pprev = &task->task_works;
        struct callback_head *work;
        unsigned long flags;

        if (likely(!task_work_pending(task)))
                return NULL;
        /*
         * If cmpxchg() fails we continue without updating pprev.
         * Either we raced with task_work_add() which added the
         * new entry before this work, we will find it again. Or
         * we raced with task_work_run(), *pprev == NULL/exited.
         */
        raw_spin_lock_irqsave(&task->pi_lock, flags);
        work = READ_ONCE(*pprev);
        while (work) {
                if (!match(work, data)) {
                        pprev = &work->next;
                        work = READ_ONCE(*pprev);
                } else if (try_cmpxchg(pprev, &work, work->next))
                        break;
        }
        raw_spin_unlock_irqrestore(&task->pi_lock, flags);

        return work;
}

static bool task_work_func_match(struct callback_head *cb, void *data)
{
        return cb->func == data;
}

/**
 * task_work_cancel_func - cancel a pending work matching a function added by task_work_add()
 * @task: the task which should execute the func's work
 * @func: identifies the func to match with a work to remove
 *
 * Find the last queued pending work with ->func == @func and remove
 * it from queue.
 *
 * RETURNS:
 * The found work or NULL if not found.
 */
struct callback_head *
task_work_cancel_func(struct task_struct *task, task_work_func_t func)
{
        return task_work_cancel_match(task, task_work_func_match, func);
}

static bool task_work_match(struct callback_head *cb, void *data)
{
        return cb == data;
}

/**
 * task_work_cancel - cancel a pending work added by task_work_add()
 * @task: the task which should execute the work
 * @cb: the callback to remove if queued
 *
 * Remove a callback from a task's queue if queued.
 *
 * RETURNS:
 * True if the callback was queued and got cancelled, false otherwise.
 */
bool task_work_cancel(struct task_struct *task, struct callback_head *cb)
{
        struct callback_head *ret;

        ret = task_work_cancel_match(task, task_work_match, cb);

        return ret == cb;
}

/**
 * task_work_run - execute the works added by task_work_add()
 *
 * Flush the pending works. Should be used by the core kernel code.
 * Called before the task returns to the user-mode or stops, or when
 * it exits. In the latter case task_work_add() can no longer add the
 * new work after task_work_run() returns.
 */
void task_work_run(void)
{
        struct task_struct *task = current;
        struct callback_head *work, *head, *next;

        for (;;) {
                /*
                 * work->func() can do task_work_add(), do not set
                 * work_exited unless the list is empty.
                 */
                work = READ_ONCE(task->task_works);
                do {
                        head = NULL;
                        if (!work) {
                                if (task->flags & PF_EXITING)
                                        head = &work_exited;
                                else
                                        break;
                        }
                } while (!try_cmpxchg(&task->task_works, &work, head));

                if (!work)
                        break;
                /*
                 * Synchronize with task_work_cancel_match(). It can not remove
                 * the first entry == work, cmpxchg(task_works) must fail.
                 * But it can remove another entry from the ->next list.
                 */
                raw_spin_lock_irq(&task->pi_lock);
                raw_spin_unlock_irq(&task->pi_lock);

                do {
                        next = work->next;
                        work->func(work);
                        work = next;
                        cond_resched();
                } while (work);
        }
}























  416 



  414 



























 1101 


 1102 

 1101 




 1103 
  416 










 1112 



 1115 


 1113 
 1112 















 1111 


   93 
















  585 




  586 















  586 












  586 



  582 


   52 

















 1101 


 1105 


























































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
// SPDX-License-Identifier: GPL-2.0
/*
 * Lockless hierarchical page accounting & limiting
 *
 * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner
 */

#include <linux/page_counter.h>
#include <linux/atomic.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/sched.h>
#include <linux/bug.h>
#include <asm/page.h>

static bool track_protection(struct page_counter *c)
{
        return c->protection_support;
}

static void propagate_protected_usage(struct page_counter *c,
                                      unsigned long usage)
{
        unsigned long protected, old_protected;
        long delta;

        if (!c->parent)
                return;

        protected = min(usage, READ_ONCE(c->min));
        old_protected = atomic_long_read(&c->min_usage);
        if (protected != old_protected) {
                old_protected = atomic_long_xchg(&c->min_usage, protected);
                delta = protected - old_protected;
                if (delta)
                        atomic_long_add(delta, &c->parent->children_min_usage);
        }

        protected = min(usage, READ_ONCE(c->low));
        old_protected = atomic_long_read(&c->low_usage);
        if (protected != old_protected) {
                old_protected = atomic_long_xchg(&c->low_usage, protected);
                delta = protected - old_protected;
                if (delta)
                        atomic_long_add(delta, &c->parent->children_low_usage);
        }
}

/**
 * page_counter_cancel - take pages out of the local counter
 * @counter: counter
 * @nr_pages: number of pages to cancel
 */
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages)
{
        long new;

        new = atomic_long_sub_return(nr_pages, &counter->usage);
        /* More uncharges than charges? */
        if (WARN_ONCE(new < 0, "page_counter underflow: %ld nr_pages=%lu\n",
                      new, nr_pages)) {
                new = 0;
                atomic_long_set(&counter->usage, new);
        }
        if (track_protection(counter))
                propagate_protected_usage(counter, new);
}

/**
 * page_counter_charge - hierarchically charge pages
 * @counter: counter
 * @nr_pages: number of pages to charge
 *
 * NOTE: This does not consider any configured counter limits.
 */
void page_counter_charge(struct page_counter *counter, unsigned long nr_pages)
{
        struct page_counter *c;
        bool protection = track_protection(counter);

        for (c = counter; c; c = c->parent) {
                long new;

                new = atomic_long_add_return(nr_pages, &c->usage);
                if (protection)
                        propagate_protected_usage(c, new);
                /*
                 * This is indeed racy, but we can live with some
                 * inaccuracy in the watermark.
                 *
                 * Notably, we have two watermarks to allow for both a globally
                 * visible peak and one that can be reset at a smaller scope.
                 *
                 * Since we reset both watermarks when the global reset occurs,
                 * we can guarantee that watermark >= local_watermark, so we
                 * don't need to do both comparisons every time.
                 *
                 * On systems with branch predictors, the inner condition should
                 * be almost free.
                 */
                if (new > READ_ONCE(c->local_watermark)) {
                        WRITE_ONCE(c->local_watermark, new);
                        if (new > READ_ONCE(c->watermark))
                                WRITE_ONCE(c->watermark, new);
                }
        }
}

/**
 * page_counter_try_charge - try to hierarchically charge pages
 * @counter: counter
 * @nr_pages: number of pages to charge
 * @fail: points first counter to hit its limit, if any
 *
 * Returns %true on success, or %false and @fail if the counter or one
 * of its ancestors has hit its configured limit.
 */
bool page_counter_try_charge(struct page_counter *counter,
                             unsigned long nr_pages,
                             struct page_counter **fail)
{
        struct page_counter *c;
        bool protection = track_protection(counter);
        bool track_failcnt = counter->track_failcnt;

        for (c = counter; c; c = c->parent) {
                long new;
                /*
                 * Charge speculatively to avoid an expensive CAS.  If
                 * a bigger charge fails, it might falsely lock out a
                 * racing smaller charge and send it into reclaim
                 * early, but the error is limited to the difference
                 * between the two sizes, which is less than 2M/4M in
                 * case of a THP locking out a regular page charge.
                 *
                 * The atomic_long_add_return() implies a full memory
                 * barrier between incrementing the count and reading
                 * the limit.  When racing with page_counter_set_max(),
                 * we either see the new limit or the setter sees the
                 * counter has changed and retries.
                 */
                new = atomic_long_add_return(nr_pages, &c->usage);
                if (new > c->max) {
                        atomic_long_sub(nr_pages, &c->usage);
                        /*
                         * This is racy, but we can live with some
                         * inaccuracy in the failcnt which is only used
                         * to report stats.
                         */
                        if (track_failcnt)
                                data_race(c->failcnt++);
                        *fail = c;
                        goto failed;
                }
                if (protection)
                        propagate_protected_usage(c, new);

                /* see comment on page_counter_charge */
                if (new > READ_ONCE(c->local_watermark)) {
                        WRITE_ONCE(c->local_watermark, new);
                        if (new > READ_ONCE(c->watermark))
                                WRITE_ONCE(c->watermark, new);
                }
        }
        return true;

failed:
        for (c = counter; c != *fail; c = c->parent)
                page_counter_cancel(c, nr_pages);

        return false;
}

/**
 * page_counter_uncharge - hierarchically uncharge pages
 * @counter: counter
 * @nr_pages: number of pages to uncharge
 */
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages)
{
        struct page_counter *c;

        for (c = counter; c; c = c->parent)
                page_counter_cancel(c, nr_pages);
}

/**
 * page_counter_set_max - set the maximum number of pages allowed
 * @counter: counter
 * @nr_pages: limit to set
 *
 * Returns 0 on success, -EBUSY if the current number of pages on the
 * counter already exceeds the specified limit.
 *
 * The caller must serialize invocations on the same counter.
 */
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages)
{
        for (;;) {
                unsigned long old;
                long usage;

                /*
                 * Update the limit while making sure that it's not
                 * below the concurrently-changing counter value.
                 *
                 * The xchg implies two full memory barriers before
                 * and after, so the read-swap-read is ordered and
                 * ensures coherency with page_counter_try_charge():
                 * that function modifies the count before checking
                 * the limit, so if it sees the old limit, we see the
                 * modified counter and retry.
                 */
                usage = page_counter_read(counter);

                if (usage > nr_pages)
                        return -EBUSY;

                old = xchg(&counter->max, nr_pages);

                if (page_counter_read(counter) <= usage || nr_pages >= old)
                        return 0;

                counter->max = old;
                cond_resched();
        }
}

/**
 * page_counter_set_min - set the amount of protected memory
 * @counter: counter
 * @nr_pages: value to set
 *
 * The caller must serialize invocations on the same counter.
 */
void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages)
{
        struct page_counter *c;

        WRITE_ONCE(counter->min, nr_pages);

        for (c = counter; c; c = c->parent)
                propagate_protected_usage(c, atomic_long_read(&c->usage));
}

/**
 * page_counter_set_low - set the amount of protected memory
 * @counter: counter
 * @nr_pages: value to set
 *
 * The caller must serialize invocations on the same counter.
 */
void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages)
{
        struct page_counter *c;

        WRITE_ONCE(counter->low, nr_pages);

        for (c = counter; c; c = c->parent)
                propagate_protected_usage(c, atomic_long_read(&c->usage));
}

/**
 * page_counter_memparse - memparse() for page counter limits
 * @buf: string to parse
 * @max: string meaning maximum possible value
 * @nr_pages: returns the result in number of pages
 *
 * Returns -EINVAL, or 0 and @nr_pages on success.  @nr_pages will be
 * limited to %PAGE_COUNTER_MAX.
 */
int page_counter_memparse(const char *buf, const char *max,
                          unsigned long *nr_pages)
{
        char *end;
        u64 bytes;

        if (!strcmp(buf, max)) {
                *nr_pages = PAGE_COUNTER_MAX;
                return 0;
        }

        bytes = memparse(buf, &end);
        if (*end != '\0')
                return -EINVAL;

        *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX);

        return 0;
}


#if IS_ENABLED(CONFIG_MEMCG) || IS_ENABLED(CONFIG_CGROUP_DMEM)
/*
 * This function calculates an individual page counter's effective
 * protection which is derived from its own memory.min/low, its
 * parent's and siblings' settings, as well as the actual memory
 * distribution in the tree.
 *
 * The following rules apply to the effective protection values:
 *
 * 1. At the first level of reclaim, effective protection is equal to
 *    the declared protection in memory.min and memory.low.
 *
 * 2. To enable safe delegation of the protection configuration, at
 *    subsequent levels the effective protection is capped to the
 *    parent's effective protection.
 *
 * 3. To make complex and dynamic subtrees easier to configure, the
 *    user is allowed to overcommit the declared protection at a given
 *    level. If that is the case, the parent's effective protection is
 *    distributed to the children in proportion to how much protection
 *    they have declared and how much of it they are utilizing.
 *
 *    This makes distribution proportional, but also work-conserving:
 *    if one counter claims much more protection than it uses memory,
 *    the unused remainder is available to its siblings.
 *
 * 4. Conversely, when the declared protection is undercommitted at a
 *    given level, the distribution of the larger parental protection
 *    budget is NOT proportional. A counter's protection from a sibling
 *    is capped to its own memory.min/low setting.
 *
 * 5. However, to allow protecting recursive subtrees from each other
 *    without having to declare each individual counter's fixed share
 *    of the ancestor's claim to protection, any unutilized -
 *    "floating" - protection from up the tree is distributed in
 *    proportion to each counter's *usage*. This makes the protection
 *    neutral wrt sibling cgroups and lets them compete freely over
 *    the shared parental protection budget, but it protects the
 *    subtree as a whole from neighboring subtrees.
 *
 * Note that 4. and 5. are not in conflict: 4. is about protecting
 * against immediate siblings whereas 5. is about protecting against
 * neighboring subtrees.
 */
static unsigned long effective_protection(unsigned long usage,
                                          unsigned long parent_usage,
                                          unsigned long setting,
                                          unsigned long parent_effective,
                                          unsigned long siblings_protected,
                                          bool recursive_protection)
{
        unsigned long protected;
        unsigned long ep;

        protected = min(usage, setting);
        /*
         * If all cgroups at this level combined claim and use more
         * protection than what the parent affords them, distribute
         * shares in proportion to utilization.
         *
         * We are using actual utilization rather than the statically
         * claimed protection in order to be work-conserving: claimed
         * but unused protection is available to siblings that would
         * otherwise get a smaller chunk than what they claimed.
         */
        if (siblings_protected > parent_effective)
                return protected * parent_effective / siblings_protected;

        /*
         * Ok, utilized protection of all children is within what the
         * parent affords them, so we know whatever this child claims
         * and utilizes is effectively protected.
         *
         * If there is unprotected usage beyond this value, reclaim
         * will apply pressure in proportion to that amount.
         *
         * If there is unutilized protection, the cgroup will be fully
         * shielded from reclaim, but we do return a smaller value for
         * protection than what the group could enjoy in theory. This
         * is okay. With the overcommit distribution above, effective
         * protection is always dependent on how memory is actually
         * consumed among the siblings anyway.
         */
        ep = protected;

        /*
         * If the children aren't claiming (all of) the protection
         * afforded to them by the parent, distribute the remainder in
         * proportion to the (unprotected) memory of each cgroup. That
         * way, cgroups that aren't explicitly prioritized wrt each
         * other compete freely over the allowance, but they are
         * collectively protected from neighboring trees.
         *
         * We're using unprotected memory for the weight so that if
         * some cgroups DO claim explicit protection, we don't protect
         * the same bytes twice.
         *
         * Check both usage and parent_usage against the respective
         * protected values. One should imply the other, but they
         * aren't read atomically - make sure the division is sane.
         */
        if (!recursive_protection)
                return ep;

        if (parent_effective > siblings_protected &&
            parent_usage > siblings_protected &&
            usage > protected) {
                unsigned long unclaimed;

                unclaimed = parent_effective - siblings_protected;
                unclaimed *= usage - protected;
                unclaimed /= parent_usage - siblings_protected;

                ep += unclaimed;
        }

        return ep;
}


/**
 * page_counter_calculate_protection - check if memory consumption is in the normal range
 * @root: the top ancestor of the sub-tree being checked
 * @counter: the page_counter the counter to update
 * @recursive_protection: Whether to use memory_recursiveprot behavior.
 *
 * Calculates elow/emin thresholds for given page_counter.
 *
 * WARNING: This function is not stateless! It can only be used as part
 *          of a top-down tree iteration, not for isolated queries.
 */
void page_counter_calculate_protection(struct page_counter *root,
                                       struct page_counter *counter,
                                       bool recursive_protection)
{
        unsigned long usage, parent_usage;
        struct page_counter *parent = counter->parent;

        /*
         * Effective values of the reclaim targets are ignored so they
         * can be stale. Have a look at mem_cgroup_protection for more
         * details.
         * TODO: calculation should be more robust so that we do not need
         * that special casing.
         */
        if (root == counter)
                return;

        usage = page_counter_read(counter);
        if (!usage)
                return;

        if (parent == root) {
                counter->emin = READ_ONCE(counter->min);
                counter->elow = READ_ONCE(counter->low);
                return;
        }

        parent_usage = page_counter_read(parent);

        WRITE_ONCE(counter->emin, effective_protection(usage, parent_usage,
                        READ_ONCE(counter->min),
                        READ_ONCE(parent->emin),
                        atomic_long_read(&parent->children_min_usage),
                        recursive_protection));

        WRITE_ONCE(counter->elow, effective_protection(usage, parent_usage,
                        READ_ONCE(counter->low),
                        READ_ONCE(parent->elow),
                        atomic_long_read(&parent->children_low_usage),
                        recursive_protection));
}
#endif /* CONFIG_MEMCG || CONFIG_CGROUP_DMEM */












































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    6 






    5 
    6 






































    6 


    6 






































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                PACKET - implements raw packet sockets.
 *
 * Authors:        Ross Biro
 *                Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 *                Alan Cox, <gw4pts@gw4pts.ampr.org>
 *
 * Fixes:
 *                Alan Cox        :        verify_area() now used correctly
 *                Alan Cox        :        new skbuff lists, look ma no backlogs!
 *                Alan Cox        :        tidied skbuff lists.
 *                Alan Cox        :        Now uses generic datagram routines I
 *                                        added. Also fixed the peek/read crash
 *                                        from all old Linux datagram code.
 *                Alan Cox        :        Uses the improved datagram code.
 *                Alan Cox        :        Added NULL's for socket options.
 *                Alan Cox        :        Re-commented the code.
 *                Alan Cox        :        Use new kernel side addressing
 *                Rob Janssen        :        Correct MTU usage.
 *                Dave Platt        :        Counter leaks caused by incorrect
 *                                        interrupt locking and some slightly
 *                                        dubious gcc output. Can you read
 *                                        compiler: it said _VOLATILE_
 *        Richard Kooijman        :        Timestamp fixes.
 *                Alan Cox        :        New buffers. Use sk->mac.raw.
 *                Alan Cox        :        sendmsg/recvmsg support.
 *                Alan Cox        :        Protocol setting support
 *        Alexey Kuznetsov        :        Untied from IPv4 stack.
 *        Cyrus Durgin                :        Fixed kerneld for kmod.
 *        Michal Ostrowski        :       Module initialization cleanup.
 *         Ulises Alonso        :       Frame number limit removal and
 *                                      packet_set_ring memory leak.
 *                Eric Biederman        :        Allow for > 8 byte hardware addresses.
 *                                        The convention is that longer addresses
 *                                        will simply extend the hardware address
 *                                        byte arrays at the end of sockaddr_ll
 *                                        and packet_mreq.
 *                Johann Baudy        :        Added TX RING.
 *                Chetan Loke        :        Implemented TPACKET_V3 block abstraction
 *                                        layer.
 *                                        Copyright (C) 2011, <lokec@ccs.neu.edu>
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/ethtool.h>
#include <linux/filter.h>
#include <linux/types.h>
#include <linux/mm.h>
#include <linux/capability.h>
#include <linux/fcntl.h>
#include <linux/socket.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/if_packet.h>
#include <linux/wireless.h>
#include <linux/kernel.h>
#include <linux/kmod.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <net/net_namespace.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/errno.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
#include <asm/ioctls.h>
#include <asm/page.h>
#include <asm/cacheflush.h>
#include <asm/io.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/poll.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/if_vlan.h>
#include <linux/virtio_net.h>
#include <linux/errqueue.h>
#include <linux/net_tstamp.h>
#include <linux/percpu.h>
#ifdef CONFIG_INET
#include <net/inet_common.h>
#endif
#include <linux/bpf.h>
#include <net/compat.h>
#include <linux/netfilter_netdev.h>

#include "internal.h"

/*
   Assumptions:
   - If the device has no dev->header_ops->create, there is no LL header
     visible above the device. In this case, its hard_header_len should be 0.
     The device may prepend its own header internally. In this case, its
     needed_headroom should be set to the space needed for it to add its
     internal header.
     For example, a WiFi driver pretending to be an Ethernet driver should
     set its hard_header_len to be the Ethernet header length, and set its
     needed_headroom to be (the real WiFi header length - the fake Ethernet
     header length).
   - packet socket receives packets with pulled ll header,
     so that SOCK_RAW should push it back.

On receive:
-----------

Incoming, dev_has_header(dev) == true
   mac_header -> ll header
   data       -> data

Outgoing, dev_has_header(dev) == true
   mac_header -> ll header
   data       -> ll header

Incoming, dev_has_header(dev) == false
   mac_header -> data
     However drivers often make it point to the ll header.
     This is incorrect because the ll header should be invisible to us.
   data       -> data

Outgoing, dev_has_header(dev) == false
   mac_header -> data. ll header is invisible to us.
   data       -> data

Resume
  If dev_has_header(dev) == false we are unable to restore the ll header,
    because it is invisible to us.


On transmit:
------------

dev_has_header(dev) == true
   mac_header -> ll header
   data       -> ll header

dev_has_header(dev) == false (ll header is invisible to us)
   mac_header -> data
   data       -> data

   We should set network_header on output to the correct position,
   packet classifier depends on it.
 */

/* Private packet socket structures. */

/* identical to struct packet_mreq except it has
 * a longer address field.
 */
struct packet_mreq_max {
        int                mr_ifindex;
        unsigned short        mr_type;
        unsigned short        mr_alen;
        unsigned char        mr_address[MAX_ADDR_LEN];
};

union tpacket_uhdr {
        struct tpacket_hdr  *h1;
        struct tpacket2_hdr *h2;
        struct tpacket3_hdr *h3;
        void *raw;
};

static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
                int closing, int tx_ring);

#define V3_ALIGNMENT        (8)

#define BLK_HDR_LEN        (ALIGN(sizeof(struct tpacket_block_desc), V3_ALIGNMENT))

#define BLK_PLUS_PRIV(sz_of_priv) \
        (BLK_HDR_LEN + ALIGN((sz_of_priv), V3_ALIGNMENT))

#define BLOCK_STATUS(x)        ((x)->hdr.bh1.block_status)
#define BLOCK_NUM_PKTS(x)        ((x)->hdr.bh1.num_pkts)
#define BLOCK_O2FP(x)                ((x)->hdr.bh1.offset_to_first_pkt)
#define BLOCK_LEN(x)                ((x)->hdr.bh1.blk_len)
#define BLOCK_SNUM(x)                ((x)->hdr.bh1.seq_num)
#define BLOCK_O2PRIV(x)        ((x)->offset_to_priv)

struct packet_sock;
static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
                       struct packet_type *pt, struct net_device *orig_dev);

static void *packet_previous_frame(struct packet_sock *po,
                struct packet_ring_buffer *rb,
                int status);
static void packet_increment_head(struct packet_ring_buffer *buff);
static int prb_curr_blk_in_use(struct tpacket_block_desc *);
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *,
                        struct packet_sock *);
static void prb_retire_current_block(struct tpacket_kbdq_core *,
                struct packet_sock *, unsigned int status);
static int prb_queue_frozen(struct tpacket_kbdq_core *);
static void prb_open_block(struct tpacket_kbdq_core *,
                struct tpacket_block_desc *);
static void prb_retire_rx_blk_timer_expired(struct timer_list *);
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *);
static void prb_fill_rxhash(struct tpacket_kbdq_core *, struct tpacket3_hdr *);
static void prb_clear_rxhash(struct tpacket_kbdq_core *,
                struct tpacket3_hdr *);
static void prb_fill_vlan_info(struct tpacket_kbdq_core *,
                struct tpacket3_hdr *);
static void packet_flush_mclist(struct sock *sk);
static u16 packet_pick_tx_queue(struct sk_buff *skb);

struct packet_skb_cb {
        union {
                struct sockaddr_pkt pkt;
                union {
                        /* Trick: alias skb original length with
                         * ll.sll_family and ll.protocol in order
                         * to save room.
                         */
                        unsigned int origlen;
                        struct sockaddr_ll ll;
                };
        } sa;
};

#define vio_le() virtio_legacy_is_little_endian()

#define PACKET_SKB_CB(__skb)        ((struct packet_skb_cb *)((__skb)->cb))

#define GET_PBDQC_FROM_RB(x)        ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
#define GET_PBLOCK_DESC(x, bid)        \
        ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
#define GET_CURR_PBLOCK_DESC_FROM_CORE(x)        \
        ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
#define GET_NEXT_PRB_BLK_NUM(x) \
        (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
        ((x)->kactive_blk_num+1) : 0)

static void __fanout_unlink(struct sock *sk, struct packet_sock *po);
static void __fanout_link(struct sock *sk, struct packet_sock *po);

#ifdef CONFIG_NETFILTER_EGRESS
static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
{
        struct sk_buff *next, *head = NULL, *tail;
        int rc;

        rcu_read_lock();
        for (; skb != NULL; skb = next) {
                next = skb->next;
                skb_mark_not_on_list(skb);

                if (!nf_hook_egress(skb, &rc, skb->dev))
                        continue;

                if (!head)
                        head = skb;
                else
                        tail->next = skb;

                tail = skb;
        }
        rcu_read_unlock();

        return head;
}
#endif

static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
{
        if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS))
                return dev_queue_xmit(skb);

#ifdef CONFIG_NETFILTER_EGRESS
        if (nf_hook_egress_active()) {
                skb = nf_hook_direct_egress(skb);
                if (!skb)
                        return NET_XMIT_DROP;
        }
#endif
        return dev_direct_xmit(skb, packet_pick_tx_queue(skb));
}

static struct net_device *packet_cached_dev_get(struct packet_sock *po)
{
        struct net_device *dev;

        rcu_read_lock();
        dev = rcu_dereference(po->cached_dev);
        dev_hold(dev);
        rcu_read_unlock();

        return dev;
}

static void packet_cached_dev_assign(struct packet_sock *po,
                                     struct net_device *dev)
{
        rcu_assign_pointer(po->cached_dev, dev);
}

static void packet_cached_dev_reset(struct packet_sock *po)
{
        RCU_INIT_POINTER(po->cached_dev, NULL);
}

static u16 packet_pick_tx_queue(struct sk_buff *skb)
{
        struct net_device *dev = skb->dev;
        const struct net_device_ops *ops = dev->netdev_ops;
        int cpu = raw_smp_processor_id();
        u16 queue_index;

#ifdef CONFIG_XPS
        skb->sender_cpu = cpu + 1;
#endif
        skb_record_rx_queue(skb, cpu % dev->real_num_tx_queues);
        if (ops->ndo_select_queue) {
                queue_index = ops->ndo_select_queue(dev, skb, NULL);
                queue_index = netdev_cap_txqueue(dev, queue_index);
        } else {
                queue_index = netdev_pick_tx(dev, skb, NULL);
        }

        return queue_index;
}

/* __register_prot_hook must be invoked through register_prot_hook
 * or from a context in which asynchronous accesses to the packet
 * socket is not possible (packet_create()).
 */
static void __register_prot_hook(struct sock *sk)
{
        struct packet_sock *po = pkt_sk(sk);

        if (!packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
                if (po->fanout)
                        __fanout_link(sk, po);
                else
                        dev_add_pack(&po->prot_hook);

                sock_hold(sk);
                packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 1);
        }
}

static void register_prot_hook(struct sock *sk)
{
        lockdep_assert_held_once(&pkt_sk(sk)->bind_lock);
        __register_prot_hook(sk);
}

/* If the sync parameter is true, we will temporarily drop
 * the po->bind_lock and do a synchronize_net to make sure no
 * asynchronous packet processing paths still refer to the elements
 * of po->prot_hook.  If the sync parameter is false, it is the
 * callers responsibility to take care of this.
 */
static void __unregister_prot_hook(struct sock *sk, bool sync)
{
        struct packet_sock *po = pkt_sk(sk);

        lockdep_assert_held_once(&po->bind_lock);

        packet_sock_flag_set(po, PACKET_SOCK_RUNNING, 0);

        if (po->fanout)
                __fanout_unlink(sk, po);
        else
                __dev_remove_pack(&po->prot_hook);

        __sock_put(sk);

        if (sync) {
                spin_unlock(&po->bind_lock);
                synchronize_net();
                spin_lock(&po->bind_lock);
        }
}

static void unregister_prot_hook(struct sock *sk, bool sync)
{
        struct packet_sock *po = pkt_sk(sk);

        if (packet_sock_flag(po, PACKET_SOCK_RUNNING))
                __unregister_prot_hook(sk, sync);
}

static inline struct page * __pure pgv_to_page(void *addr)
{
        if (is_vmalloc_addr(addr))
                return vmalloc_to_page(addr);
        return virt_to_page(addr);
}

static void __packet_set_status(struct packet_sock *po, void *frame, int status)
{
        union tpacket_uhdr h;

        /* WRITE_ONCE() are paired with READ_ONCE() in __packet_get_status */

        h.raw = frame;
        switch (po->tp_version) {
        case TPACKET_V1:
                WRITE_ONCE(h.h1->tp_status, status);
                flush_dcache_page(pgv_to_page(&h.h1->tp_status));
                break;
        case TPACKET_V2:
                WRITE_ONCE(h.h2->tp_status, status);
                flush_dcache_page(pgv_to_page(&h.h2->tp_status));
                break;
        case TPACKET_V3:
                WRITE_ONCE(h.h3->tp_status, status);
                flush_dcache_page(pgv_to_page(&h.h3->tp_status));
                break;
        default:
                WARN(1, "TPACKET version not supported.\n");
                BUG();
        }

        smp_wmb();
}

static int __packet_get_status(const struct packet_sock *po, void *frame)
{
        union tpacket_uhdr h;

        smp_rmb();

        /* READ_ONCE() are paired with WRITE_ONCE() in __packet_set_status */

        h.raw = frame;
        switch (po->tp_version) {
        case TPACKET_V1:
                flush_dcache_page(pgv_to_page(&h.h1->tp_status));
                return READ_ONCE(h.h1->tp_status);
        case TPACKET_V2:
                flush_dcache_page(pgv_to_page(&h.h2->tp_status));
                return READ_ONCE(h.h2->tp_status);
        case TPACKET_V3:
                flush_dcache_page(pgv_to_page(&h.h3->tp_status));
                return READ_ONCE(h.h3->tp_status);
        default:
                WARN(1, "TPACKET version not supported.\n");
                BUG();
                return 0;
        }
}

static __u32 tpacket_get_timestamp(struct sk_buff *skb, struct timespec64 *ts,
                                   unsigned int flags)
{
        struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);

        if (shhwtstamps &&
            (flags & SOF_TIMESTAMPING_RAW_HARDWARE) &&
            ktime_to_timespec64_cond(shhwtstamps->hwtstamp, ts))
                return TP_STATUS_TS_RAW_HARDWARE;

        if ((flags & SOF_TIMESTAMPING_SOFTWARE) &&
            ktime_to_timespec64_cond(skb_tstamp(skb), ts))
                return TP_STATUS_TS_SOFTWARE;

        return 0;
}

static __u32 __packet_set_timestamp(struct packet_sock *po, void *frame,
                                    struct sk_buff *skb)
{
        union tpacket_uhdr h;
        struct timespec64 ts;
        __u32 ts_status;

        if (!(ts_status = tpacket_get_timestamp(skb, &ts, READ_ONCE(po->tp_tstamp))))
                return 0;

        h.raw = frame;
        /*
         * versions 1 through 3 overflow the timestamps in y2106, since they
         * all store the seconds in a 32-bit unsigned integer.
         * If we create a version 4, that should have a 64-bit timestamp,
         * either 64-bit seconds + 32-bit nanoseconds, or just 64-bit
         * nanoseconds.
         */
        switch (po->tp_version) {
        case TPACKET_V1:
                h.h1->tp_sec = ts.tv_sec;
                h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
                break;
        case TPACKET_V2:
                h.h2->tp_sec = ts.tv_sec;
                h.h2->tp_nsec = ts.tv_nsec;
                break;
        case TPACKET_V3:
                h.h3->tp_sec = ts.tv_sec;
                h.h3->tp_nsec = ts.tv_nsec;
                break;
        default:
                WARN(1, "TPACKET version not supported.\n");
                BUG();
        }

        /* one flush is safe, as both fields always lie on the same cacheline */
        flush_dcache_page(pgv_to_page(&h.h1->tp_sec));
        smp_wmb();

        return ts_status;
}

static void *packet_lookup_frame(const struct packet_sock *po,
                                 const struct packet_ring_buffer *rb,
                                 unsigned int position,
                                 int status)
{
        unsigned int pg_vec_pos, frame_offset;
        union tpacket_uhdr h;

        pg_vec_pos = position / rb->frames_per_block;
        frame_offset = position % rb->frames_per_block;

        h.raw = rb->pg_vec[pg_vec_pos].buffer +
                (frame_offset * rb->frame_size);

        if (status != __packet_get_status(po, h.raw))
                return NULL;

        return h.raw;
}

static void *packet_current_frame(struct packet_sock *po,
                struct packet_ring_buffer *rb,
                int status)
{
        return packet_lookup_frame(po, rb, rb->head, status);
}

static u16 vlan_get_tci(const struct sk_buff *skb, struct net_device *dev)
{
        struct vlan_hdr vhdr, *vh;
        unsigned int header_len;

        if (!dev)
                return 0;

        /* In the SOCK_DGRAM scenario, skb data starts at the network
         * protocol, which is after the VLAN headers. The outer VLAN
         * header is at the hard_header_len offset in non-variable
         * length link layer headers. If it's a VLAN device, the
         * min_header_len should be used to exclude the VLAN header
         * size.
         */
        if (dev->min_header_len == dev->hard_header_len)
                header_len = dev->hard_header_len;
        else if (is_vlan_dev(dev))
                header_len = dev->min_header_len;
        else
                return 0;

        vh = skb_header_pointer(skb, skb_mac_offset(skb) + header_len,
                                sizeof(vhdr), &vhdr);
        if (unlikely(!vh))
                return 0;

        return ntohs(vh->h_vlan_TCI);
}

static __be16 vlan_get_protocol_dgram(const struct sk_buff *skb)
{
        __be16 proto = skb->protocol;

        if (unlikely(eth_type_vlan(proto)))
                proto = __vlan_get_protocol_offset(skb, proto,
                                                   skb_mac_offset(skb), NULL);

        return proto;
}

static void prb_del_retire_blk_timer(struct tpacket_kbdq_core *pkc)
{
        timer_delete_sync(&pkc->retire_blk_timer);
}

static void prb_shutdown_retire_blk_timer(struct packet_sock *po,
                struct sk_buff_head *rb_queue)
{
        struct tpacket_kbdq_core *pkc;

        pkc = GET_PBDQC_FROM_RB(&po->rx_ring);

        spin_lock_bh(&rb_queue->lock);
        pkc->delete_blk_timer = 1;
        spin_unlock_bh(&rb_queue->lock);

        prb_del_retire_blk_timer(pkc);
}

static void prb_setup_retire_blk_timer(struct packet_sock *po)
{
        struct tpacket_kbdq_core *pkc;

        pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
        timer_setup(&pkc->retire_blk_timer, prb_retire_rx_blk_timer_expired,
                    0);
        pkc->retire_blk_timer.expires = jiffies;
}

static int prb_calc_retire_blk_tmo(struct packet_sock *po,
                                int blk_size_in_bytes)
{
        struct net_device *dev;
        unsigned int mbits, div;
        struct ethtool_link_ksettings ecmd;
        int err;

        rtnl_lock();
        dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex);
        if (unlikely(!dev)) {
                rtnl_unlock();
                return DEFAULT_PRB_RETIRE_TOV;
        }
        err = __ethtool_get_link_ksettings(dev, &ecmd);
        rtnl_unlock();
        if (err)
                return DEFAULT_PRB_RETIRE_TOV;

        /* If the link speed is so slow you don't really
         * need to worry about perf anyways
         */
        if (ecmd.base.speed < SPEED_1000 ||
            ecmd.base.speed == SPEED_UNKNOWN)
                return DEFAULT_PRB_RETIRE_TOV;

        div = ecmd.base.speed / 1000;
        mbits = (blk_size_in_bytes * 8) / (1024 * 1024);

        if (div)
                mbits /= div;

        if (div)
                return mbits + 1;
        return mbits;
}

static void prb_init_ft_ops(struct tpacket_kbdq_core *p1,
                        union tpacket_req_u *req_u)
{
        p1->feature_req_word = req_u->req3.tp_feature_req_word;
}

static void init_prb_bdqc(struct packet_sock *po,
                        struct packet_ring_buffer *rb,
                        struct pgv *pg_vec,
                        union tpacket_req_u *req_u)
{
        struct tpacket_kbdq_core *p1 = GET_PBDQC_FROM_RB(rb);
        struct tpacket_block_desc *pbd;

        memset(p1, 0x0, sizeof(*p1));

        p1->knxt_seq_num = 1;
        p1->pkbdq = pg_vec;
        pbd = (struct tpacket_block_desc *)pg_vec[0].buffer;
        p1->pkblk_start        = pg_vec[0].buffer;
        p1->kblk_size = req_u->req3.tp_block_size;
        p1->knum_blocks        = req_u->req3.tp_block_nr;
        p1->hdrlen = po->tp_hdrlen;
        p1->version = po->tp_version;
        p1->last_kactive_blk_num = 0;
        po->stats.stats3.tp_freeze_q_cnt = 0;
        if (req_u->req3.tp_retire_blk_tov)
                p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov;
        else
                p1->retire_blk_tov = prb_calc_retire_blk_tmo(po,
                                                req_u->req3.tp_block_size);
        p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov);
        p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv;
        rwlock_init(&p1->blk_fill_in_prog_lock);

        p1->max_frame_len = p1->kblk_size - BLK_PLUS_PRIV(p1->blk_sizeof_priv);
        prb_init_ft_ops(p1, req_u);
        prb_setup_retire_blk_timer(po);
        prb_open_block(p1, pbd);
}

/*  Do NOT update the last_blk_num first.
 *  Assumes sk_buff_head lock is held.
 */
static void _prb_refresh_rx_retire_blk_timer(struct tpacket_kbdq_core *pkc)
{
        mod_timer(&pkc->retire_blk_timer,
                        jiffies + pkc->tov_in_jiffies);
        pkc->last_kactive_blk_num = pkc->kactive_blk_num;
}

/*
 * Timer logic:
 * 1) We refresh the timer only when we open a block.
 *    By doing this we don't waste cycles refreshing the timer
 *          on packet-by-packet basis.
 *
 * With a 1MB block-size, on a 1Gbps line, it will take
 * i) ~8 ms to fill a block + ii) memcpy etc.
 * In this cut we are not accounting for the memcpy time.
 *
 * So, if the user sets the 'tmo' to 10ms then the timer
 * will never fire while the block is still getting filled
 * (which is what we want). However, the user could choose
 * to close a block early and that's fine.
 *
 * But when the timer does fire, we check whether or not to refresh it.
 * Since the tmo granularity is in msecs, it is not too expensive
 * to refresh the timer, lets say every '8' msecs.
 * Either the user can set the 'tmo' or we can derive it based on
 * a) line-speed and b) block-size.
 * prb_calc_retire_blk_tmo() calculates the tmo.
 *
 */
static void prb_retire_rx_blk_timer_expired(struct timer_list *t)
{
        struct packet_sock *po =
                timer_container_of(po, t, rx_ring.prb_bdqc.retire_blk_timer);
        struct tpacket_kbdq_core *pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
        unsigned int frozen;
        struct tpacket_block_desc *pbd;

        spin_lock(&po->sk.sk_receive_queue.lock);

        frozen = prb_queue_frozen(pkc);
        pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

        if (unlikely(pkc->delete_blk_timer))
                goto out;

        /* We only need to plug the race when the block is partially filled.
         * tpacket_rcv:
         *                lock(); increment BLOCK_NUM_PKTS; unlock()
         *                copy_bits() is in progress ...
         *                timer fires on other cpu:
         *                we can't retire the current block because copy_bits
         *                is in progress.
         *
         */
        if (BLOCK_NUM_PKTS(pbd)) {
                /* Waiting for skb_copy_bits to finish... */
                write_lock(&pkc->blk_fill_in_prog_lock);
                write_unlock(&pkc->blk_fill_in_prog_lock);
        }

        if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) {
                if (!frozen) {
                        if (!BLOCK_NUM_PKTS(pbd)) {
                                /* An empty block. Just refresh the timer. */
                                goto refresh_timer;
                        }
                        prb_retire_current_block(pkc, po, TP_STATUS_BLK_TMO);
                        if (!prb_dispatch_next_block(pkc, po))
                                goto refresh_timer;
                        else
                                goto out;
                } else {
                        /* Case 1. Queue was frozen because user-space was
                         *           lagging behind.
                         */
                        if (prb_curr_blk_in_use(pbd)) {
                                /*
                                 * Ok, user-space is still behind.
                                 * So just refresh the timer.
                                 */
                                goto refresh_timer;
                        } else {
                               /* Case 2. queue was frozen,user-space caught up,
                                * now the link went idle && the timer fired.
                                * We don't have a block to close.So we open this
                                * block and restart the timer.
                                * opening a block thaws the queue,restarts timer
                                * Thawing/timer-refresh is a side effect.
                                */
                                prb_open_block(pkc, pbd);
                                goto out;
                        }
                }
        }

refresh_timer:
        _prb_refresh_rx_retire_blk_timer(pkc);

out:
        spin_unlock(&po->sk.sk_receive_queue.lock);
}

static void prb_flush_block(struct tpacket_kbdq_core *pkc1,
                struct tpacket_block_desc *pbd1, __u32 status)
{
        /* Flush everything minus the block header */

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
        u8 *start, *end;

        start = (u8 *)pbd1;

        /* Skip the block header(we know header WILL fit in 4K) */
        start += PAGE_SIZE;

        end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end);
        for (; start < end; start += PAGE_SIZE)
                flush_dcache_page(pgv_to_page(start));

        smp_wmb();
#endif

        /* Now update the block status. */

        BLOCK_STATUS(pbd1) = status;

        /* Flush the block header */

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
        start = (u8 *)pbd1;
        flush_dcache_page(pgv_to_page(start));

        smp_wmb();
#endif
}

/*
 * Side effect:
 *
 * 1) flush the block
 * 2) Increment active_blk_num
 *
 * Note:We DONT refresh the timer on purpose.
 *        Because almost always the next block will be opened.
 */
static void prb_close_block(struct tpacket_kbdq_core *pkc1,
                struct tpacket_block_desc *pbd1,
                struct packet_sock *po, unsigned int stat)
{
        __u32 status = TP_STATUS_USER | stat;

        struct tpacket3_hdr *last_pkt;
        struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;
        struct sock *sk = &po->sk;

        if (atomic_read(&po->tp_drops))
                status |= TP_STATUS_LOSING;

        last_pkt = (struct tpacket3_hdr *)pkc1->prev;
        last_pkt->tp_next_offset = 0;

        /* Get the ts of the last pkt */
        if (BLOCK_NUM_PKTS(pbd1)) {
                h1->ts_last_pkt.ts_sec = last_pkt->tp_sec;
                h1->ts_last_pkt.ts_nsec        = last_pkt->tp_nsec;
        } else {
                /* Ok, we tmo'd - so get the current time.
                 *
                 * It shouldn't really happen as we don't close empty
                 * blocks. See prb_retire_rx_blk_timer_expired().
                 */
                struct timespec64 ts;
                ktime_get_real_ts64(&ts);
                h1->ts_last_pkt.ts_sec = ts.tv_sec;
                h1->ts_last_pkt.ts_nsec        = ts.tv_nsec;
        }

        smp_wmb();

        /* Flush the block */
        prb_flush_block(pkc1, pbd1, status);

        sk->sk_data_ready(sk);

        pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1);
}

static void prb_thaw_queue(struct tpacket_kbdq_core *pkc)
{
        pkc->reset_pending_on_curr_blk = 0;
}

/*
 * Side effect of opening a block:
 *
 * 1) prb_queue is thawed.
 * 2) retire_blk_timer is refreshed.
 *
 */
static void prb_open_block(struct tpacket_kbdq_core *pkc1,
        struct tpacket_block_desc *pbd1)
{
        struct timespec64 ts;
        struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1;

        smp_rmb();

        /* We could have just memset this but we will lose the
         * flexibility of making the priv area sticky
         */

        BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++;
        BLOCK_NUM_PKTS(pbd1) = 0;
        BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);

        ktime_get_real_ts64(&ts);

        h1->ts_first_pkt.ts_sec = ts.tv_sec;
        h1->ts_first_pkt.ts_nsec = ts.tv_nsec;

        pkc1->pkblk_start = (char *)pbd1;
        pkc1->nxt_offset = pkc1->pkblk_start + BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);

        BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv);
        BLOCK_O2PRIV(pbd1) = BLK_HDR_LEN;

        pbd1->version = pkc1->version;
        pkc1->prev = pkc1->nxt_offset;
        pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size;

        prb_thaw_queue(pkc1);
        _prb_refresh_rx_retire_blk_timer(pkc1);

        smp_wmb();
}

/*
 * Queue freeze logic:
 * 1) Assume tp_block_nr = 8 blocks.
 * 2) At time 't0', user opens Rx ring.
 * 3) Some time past 't0', kernel starts filling blocks starting from 0 .. 7
 * 4) user-space is either sleeping or processing block '0'.
 * 5) tpacket_rcv is currently filling block '7', since there is no space left,
 *    it will close block-7,loop around and try to fill block '0'.
 *    call-flow:
 *    __packet_lookup_frame_in_block
 *      prb_retire_current_block()
 *      prb_dispatch_next_block()
 *        |->(BLOCK_STATUS == USER) evaluates to true
 *    5.1) Since block-0 is currently in-use, we just freeze the queue.
 * 6) Now there are two cases:
 *    6.1) Link goes idle right after the queue is frozen.
 *         But remember, the last open_block() refreshed the timer.
 *         When this timer expires,it will refresh itself so that we can
 *         re-open block-0 in near future.
 *    6.2) Link is busy and keeps on receiving packets. This is a simple
 *         case and __packet_lookup_frame_in_block will check if block-0
 *         is free and can now be re-used.
 */
static void prb_freeze_queue(struct tpacket_kbdq_core *pkc,
                                  struct packet_sock *po)
{
        pkc->reset_pending_on_curr_blk = 1;
        po->stats.stats3.tp_freeze_q_cnt++;
}

#define TOTAL_PKT_LEN_INCL_ALIGN(length) (ALIGN((length), V3_ALIGNMENT))

/*
 * If the next block is free then we will dispatch it
 * and return a good offset.
 * Else, we will freeze the queue.
 * So, caller must check the return value.
 */
static void *prb_dispatch_next_block(struct tpacket_kbdq_core *pkc,
                struct packet_sock *po)
{
        struct tpacket_block_desc *pbd;

        smp_rmb();

        /* 1. Get current block num */
        pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

        /* 2. If this block is currently in_use then freeze the queue */
        if (TP_STATUS_USER & BLOCK_STATUS(pbd)) {
                prb_freeze_queue(pkc, po);
                return NULL;
        }

        /*
         * 3.
         * open this block and return the offset where the first packet
         * needs to get stored.
         */
        prb_open_block(pkc, pbd);
        return (void *)pkc->nxt_offset;
}

static void prb_retire_current_block(struct tpacket_kbdq_core *pkc,
                struct packet_sock *po, unsigned int status)
{
        struct tpacket_block_desc *pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

        /* retire/close the current block */
        if (likely(TP_STATUS_KERNEL == BLOCK_STATUS(pbd))) {
                /*
                 * Plug the case where copy_bits() is in progress on
                 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't
                 * have space to copy the pkt in the current block and
                 * called prb_retire_current_block()
                 *
                 * We don't need to worry about the TMO case because
                 * the timer-handler already handled this case.
                 */
                if (!(status & TP_STATUS_BLK_TMO)) {
                        /* Waiting for skb_copy_bits to finish... */
                        write_lock(&pkc->blk_fill_in_prog_lock);
                        write_unlock(&pkc->blk_fill_in_prog_lock);
                }
                prb_close_block(pkc, pbd, po, status);
                return;
        }
}

static int prb_curr_blk_in_use(struct tpacket_block_desc *pbd)
{
        return TP_STATUS_USER & BLOCK_STATUS(pbd);
}

static int prb_queue_frozen(struct tpacket_kbdq_core *pkc)
{
        return pkc->reset_pending_on_curr_blk;
}

static void prb_clear_blk_fill_status(struct packet_ring_buffer *rb)
        __releases(&pkc->blk_fill_in_prog_lock)
{
        struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);

        read_unlock(&pkc->blk_fill_in_prog_lock);
}

static void prb_fill_rxhash(struct tpacket_kbdq_core *pkc,
                        struct tpacket3_hdr *ppd)
{
        ppd->hv1.tp_rxhash = skb_get_hash(pkc->skb);
}

static void prb_clear_rxhash(struct tpacket_kbdq_core *pkc,
                        struct tpacket3_hdr *ppd)
{
        ppd->hv1.tp_rxhash = 0;
}

static void prb_fill_vlan_info(struct tpacket_kbdq_core *pkc,
                        struct tpacket3_hdr *ppd)
{
        struct packet_sock *po = container_of(pkc, struct packet_sock, rx_ring.prb_bdqc);

        if (skb_vlan_tag_present(pkc->skb)) {
                ppd->hv1.tp_vlan_tci = skb_vlan_tag_get(pkc->skb);
                ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->vlan_proto);
                ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
        } else if (unlikely(po->sk.sk_type == SOCK_DGRAM && eth_type_vlan(pkc->skb->protocol))) {
                ppd->hv1.tp_vlan_tci = vlan_get_tci(pkc->skb, pkc->skb->dev);
                ppd->hv1.tp_vlan_tpid = ntohs(pkc->skb->protocol);
                ppd->tp_status = TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
        } else {
                ppd->hv1.tp_vlan_tci = 0;
                ppd->hv1.tp_vlan_tpid = 0;
                ppd->tp_status = TP_STATUS_AVAILABLE;
        }
}

static void prb_run_all_ft_ops(struct tpacket_kbdq_core *pkc,
                        struct tpacket3_hdr *ppd)
{
        ppd->hv1.tp_padding = 0;
        prb_fill_vlan_info(pkc, ppd);

        if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH)
                prb_fill_rxhash(pkc, ppd);
        else
                prb_clear_rxhash(pkc, ppd);
}

static void prb_fill_curr_block(char *curr,
                                struct tpacket_kbdq_core *pkc,
                                struct tpacket_block_desc *pbd,
                                unsigned int len)
        __acquires(&pkc->blk_fill_in_prog_lock)
{
        struct tpacket3_hdr *ppd;

        ppd  = (struct tpacket3_hdr *)curr;
        ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len);
        pkc->prev = curr;
        pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len);
        BLOCK_LEN(pbd) += TOTAL_PKT_LEN_INCL_ALIGN(len);
        BLOCK_NUM_PKTS(pbd) += 1;
        read_lock(&pkc->blk_fill_in_prog_lock);
        prb_run_all_ft_ops(pkc, ppd);
}

/* Assumes caller has the sk->rx_queue.lock */
static void *__packet_lookup_frame_in_block(struct packet_sock *po,
                                            struct sk_buff *skb,
                                            unsigned int len
                                            )
{
        struct tpacket_kbdq_core *pkc;
        struct tpacket_block_desc *pbd;
        char *curr, *end;

        pkc = GET_PBDQC_FROM_RB(&po->rx_ring);
        pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);

        /* Queue is frozen when user space is lagging behind */
        if (prb_queue_frozen(pkc)) {
                /*
                 * Check if that last block which caused the queue to freeze,
                 * is still in_use by user-space.
                 */
                if (prb_curr_blk_in_use(pbd)) {
                        /* Can't record this packet */
                        return NULL;
                } else {
                        /*
                         * Ok, the block was released by user-space.
                         * Now let's open that block.
                         * opening a block also thaws the queue.
                         * Thawing is a side effect.
                         */
                        prb_open_block(pkc, pbd);
                }
        }

        smp_mb();
        curr = pkc->nxt_offset;
        pkc->skb = skb;
        end = (char *)pbd + pkc->kblk_size;

        /* first try the current block */
        if (curr+TOTAL_PKT_LEN_INCL_ALIGN(len) < end) {
                prb_fill_curr_block(curr, pkc, pbd, len);
                return (void *)curr;
        }

        /* Ok, close the current block */
        prb_retire_current_block(pkc, po, 0);

        /* Now, try to dispatch the next block */
        curr = (char *)prb_dispatch_next_block(pkc, po);
        if (curr) {
                pbd = GET_CURR_PBLOCK_DESC_FROM_CORE(pkc);
                prb_fill_curr_block(curr, pkc, pbd, len);
                return (void *)curr;
        }

        /*
         * No free blocks are available.user_space hasn't caught up yet.
         * Queue was just frozen and now this packet will get dropped.
         */
        return NULL;
}

static void *packet_current_rx_frame(struct packet_sock *po,
                                            struct sk_buff *skb,
                                            int status, unsigned int len)
{
        char *curr = NULL;
        switch (po->tp_version) {
        case TPACKET_V1:
        case TPACKET_V2:
                curr = packet_lookup_frame(po, &po->rx_ring,
                                        po->rx_ring.head, status);
                return curr;
        case TPACKET_V3:
                return __packet_lookup_frame_in_block(po, skb, len);
        default:
                WARN(1, "TPACKET version not supported\n");
                BUG();
                return NULL;
        }
}

static void *prb_lookup_block(const struct packet_sock *po,
                              const struct packet_ring_buffer *rb,
                              unsigned int idx,
                              int status)
{
        struct tpacket_kbdq_core *pkc  = GET_PBDQC_FROM_RB(rb);
        struct tpacket_block_desc *pbd = GET_PBLOCK_DESC(pkc, idx);

        if (status != BLOCK_STATUS(pbd))
                return NULL;
        return pbd;
}

static int prb_previous_blk_num(struct packet_ring_buffer *rb)
{
        unsigned int prev;
        if (rb->prb_bdqc.kactive_blk_num)
                prev = rb->prb_bdqc.kactive_blk_num-1;
        else
                prev = rb->prb_bdqc.knum_blocks-1;
        return prev;
}

/* Assumes caller has held the rx_queue.lock */
static void *__prb_previous_block(struct packet_sock *po,
                                         struct packet_ring_buffer *rb,
                                         int status)
{
        unsigned int previous = prb_previous_blk_num(rb);
        return prb_lookup_block(po, rb, previous, status);
}

static void *packet_previous_rx_frame(struct packet_sock *po,
                                             struct packet_ring_buffer *rb,
                                             int status)
{
        if (po->tp_version <= TPACKET_V2)
                return packet_previous_frame(po, rb, status);

        return __prb_previous_block(po, rb, status);
}

static void packet_increment_rx_head(struct packet_sock *po,
                                            struct packet_ring_buffer *rb)
{
        switch (po->tp_version) {
        case TPACKET_V1:
        case TPACKET_V2:
                return packet_increment_head(rb);
        case TPACKET_V3:
        default:
                WARN(1, "TPACKET version not supported.\n");
                BUG();
                return;
        }
}

static void *packet_previous_frame(struct packet_sock *po,
                struct packet_ring_buffer *rb,
                int status)
{
        unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
        return packet_lookup_frame(po, rb, previous, status);
}

static void packet_increment_head(struct packet_ring_buffer *buff)
{
        buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
}

static void packet_inc_pending(struct packet_ring_buffer *rb)
{
        this_cpu_inc(*rb->pending_refcnt);
}

static void packet_dec_pending(struct packet_ring_buffer *rb)
{
        this_cpu_dec(*rb->pending_refcnt);
}

static unsigned int packet_read_pending(const struct packet_ring_buffer *rb)
{
        unsigned int refcnt = 0;
        int cpu;

        /* We don't use pending refcount in rx_ring. */
        if (rb->pending_refcnt == NULL)
                return 0;

        for_each_possible_cpu(cpu)
                refcnt += *per_cpu_ptr(rb->pending_refcnt, cpu);

        return refcnt;
}

static int packet_alloc_pending(struct packet_sock *po)
{
        po->rx_ring.pending_refcnt = NULL;

        po->tx_ring.pending_refcnt = alloc_percpu(unsigned int);
        if (unlikely(po->tx_ring.pending_refcnt == NULL))
                return -ENOBUFS;

        return 0;
}

static void packet_free_pending(struct packet_sock *po)
{
        free_percpu(po->tx_ring.pending_refcnt);
}

#define ROOM_POW_OFF        2
#define ROOM_NONE        0x0
#define ROOM_LOW        0x1
#define ROOM_NORMAL        0x2

static bool __tpacket_has_room(const struct packet_sock *po, int pow_off)
{
        int idx, len;

        len = READ_ONCE(po->rx_ring.frame_max) + 1;
        idx = READ_ONCE(po->rx_ring.head);
        if (pow_off)
                idx += len >> pow_off;
        if (idx >= len)
                idx -= len;
        return packet_lookup_frame(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
}

static bool __tpacket_v3_has_room(const struct packet_sock *po, int pow_off)
{
        int idx, len;

        len = READ_ONCE(po->rx_ring.prb_bdqc.knum_blocks);
        idx = READ_ONCE(po->rx_ring.prb_bdqc.kactive_blk_num);
        if (pow_off)
                idx += len >> pow_off;
        if (idx >= len)
                idx -= len;
        return prb_lookup_block(po, &po->rx_ring, idx, TP_STATUS_KERNEL);
}

static int __packet_rcv_has_room(const struct packet_sock *po,
                                 const struct sk_buff *skb)
{
        const struct sock *sk = &po->sk;
        int ret = ROOM_NONE;

        if (po->prot_hook.func != tpacket_rcv) {
                int rcvbuf = READ_ONCE(sk->sk_rcvbuf);
                int avail = rcvbuf - atomic_read(&sk->sk_rmem_alloc)
                                   - (skb ? skb->truesize : 0);

                if (avail > (rcvbuf >> ROOM_POW_OFF))
                        return ROOM_NORMAL;
                else if (avail > 0)
                        return ROOM_LOW;
                else
                        return ROOM_NONE;
        }

        if (po->tp_version == TPACKET_V3) {
                if (__tpacket_v3_has_room(po, ROOM_POW_OFF))
                        ret = ROOM_NORMAL;
                else if (__tpacket_v3_has_room(po, 0))
                        ret = ROOM_LOW;
        } else {
                if (__tpacket_has_room(po, ROOM_POW_OFF))
                        ret = ROOM_NORMAL;
                else if (__tpacket_has_room(po, 0))
                        ret = ROOM_LOW;
        }

        return ret;
}

static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb)
{
        bool pressure;
        int ret;

        ret = __packet_rcv_has_room(po, skb);
        pressure = ret != ROOM_NORMAL;

        if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) != pressure)
                packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, pressure);

        return ret;
}

static void packet_rcv_try_clear_pressure(struct packet_sock *po)
{
        if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) &&
            __packet_rcv_has_room(po, NULL) == ROOM_NORMAL)
                packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false);
}

static void packet_sock_destruct(struct sock *sk)
{
        skb_queue_purge(&sk->sk_error_queue);

        WARN_ON(atomic_read(&sk->sk_rmem_alloc));
        WARN_ON(refcount_read(&sk->sk_wmem_alloc));

        if (!sock_flag(sk, SOCK_DEAD)) {
                pr_err("Attempt to release alive packet socket: %p\n", sk);
                return;
        }
}

static bool fanout_flow_is_huge(struct packet_sock *po, struct sk_buff *skb)
{
        u32 *history = po->rollover->history;
        u32 victim, rxhash;
        int i, count = 0;

        rxhash = skb_get_hash(skb);
        for (i = 0; i < ROLLOVER_HLEN; i++)
                if (READ_ONCE(history[i]) == rxhash)
                        count++;

        victim = get_random_u32_below(ROLLOVER_HLEN);

        /* Avoid dirtying the cache line if possible */
        if (READ_ONCE(history[victim]) != rxhash)
                WRITE_ONCE(history[victim], rxhash);

        return count > (ROLLOVER_HLEN >> 1);
}

static unsigned int fanout_demux_hash(struct packet_fanout *f,
                                      struct sk_buff *skb,
                                      unsigned int num)
{
        return reciprocal_scale(__skb_get_hash_symmetric(skb), num);
}

static unsigned int fanout_demux_lb(struct packet_fanout *f,
                                    struct sk_buff *skb,
                                    unsigned int num)
{
        unsigned int val = atomic_inc_return(&f->rr_cur);

        return val % num;
}

static unsigned int fanout_demux_cpu(struct packet_fanout *f,
                                     struct sk_buff *skb,
                                     unsigned int num)
{
        return smp_processor_id() % num;
}

static unsigned int fanout_demux_rnd(struct packet_fanout *f,
                                     struct sk_buff *skb,
                                     unsigned int num)
{
        return get_random_u32_below(num);
}

static unsigned int fanout_demux_rollover(struct packet_fanout *f,
                                          struct sk_buff *skb,
                                          unsigned int idx, bool try_self,
                                          unsigned int num)
{
        struct packet_sock *po, *po_next, *po_skip = NULL;
        unsigned int i, j, room = ROOM_NONE;

        po = pkt_sk(rcu_dereference(f->arr[idx]));

        if (try_self) {
                room = packet_rcv_has_room(po, skb);
                if (room == ROOM_NORMAL ||
                    (room == ROOM_LOW && !fanout_flow_is_huge(po, skb)))
                        return idx;
                po_skip = po;
        }

        i = j = min_t(int, po->rollover->sock, num - 1);
        do {
                po_next = pkt_sk(rcu_dereference(f->arr[i]));
                if (po_next != po_skip &&
                    !packet_sock_flag(po_next, PACKET_SOCK_PRESSURE) &&
                    packet_rcv_has_room(po_next, skb) == ROOM_NORMAL) {
                        if (i != j)
                                po->rollover->sock = i;
                        atomic_long_inc(&po->rollover->num);
                        if (room == ROOM_LOW)
                                atomic_long_inc(&po->rollover->num_huge);
                        return i;
                }

                if (++i == num)
                        i = 0;
        } while (i != j);

        atomic_long_inc(&po->rollover->num_failed);
        return idx;
}

static unsigned int fanout_demux_qm(struct packet_fanout *f,
                                    struct sk_buff *skb,
                                    unsigned int num)
{
        return skb_get_queue_mapping(skb) % num;
}

static unsigned int fanout_demux_bpf(struct packet_fanout *f,
                                     struct sk_buff *skb,
                                     unsigned int num)
{
        struct bpf_prog *prog;
        unsigned int ret = 0;

        rcu_read_lock();
        prog = rcu_dereference(f->bpf_prog);
        if (prog)
                ret = bpf_prog_run_clear_cb(prog, skb) % num;
        rcu_read_unlock();

        return ret;
}

static bool fanout_has_flag(struct packet_fanout *f, u16 flag)
{
        return f->flags & (flag >> 8);
}

static int packet_rcv_fanout(struct sk_buff *skb, struct net_device *dev,
                             struct packet_type *pt, struct net_device *orig_dev)
{
        struct packet_fanout *f = pt->af_packet_priv;
        unsigned int num = READ_ONCE(f->num_members);
        struct net *net = read_pnet(&f->net);
        struct packet_sock *po;
        unsigned int idx;

        if (!net_eq(dev_net(dev), net) || !num) {
                kfree_skb(skb);
                return 0;
        }

        if (fanout_has_flag(f, PACKET_FANOUT_FLAG_DEFRAG)) {
                skb = ip_check_defrag(net, skb, IP_DEFRAG_AF_PACKET);
                if (!skb)
                        return 0;
        }
        switch (f->type) {
        case PACKET_FANOUT_HASH:
        default:
                idx = fanout_demux_hash(f, skb, num);
                break;
        case PACKET_FANOUT_LB:
                idx = fanout_demux_lb(f, skb, num);
                break;
        case PACKET_FANOUT_CPU:
                idx = fanout_demux_cpu(f, skb, num);
                break;
        case PACKET_FANOUT_RND:
                idx = fanout_demux_rnd(f, skb, num);
                break;
        case PACKET_FANOUT_QM:
                idx = fanout_demux_qm(f, skb, num);
                break;
        case PACKET_FANOUT_ROLLOVER:
                idx = fanout_demux_rollover(f, skb, 0, false, num);
                break;
        case PACKET_FANOUT_CBPF:
        case PACKET_FANOUT_EBPF:
                idx = fanout_demux_bpf(f, skb, num);
                break;
        }

        if (fanout_has_flag(f, PACKET_FANOUT_FLAG_ROLLOVER))
                idx = fanout_demux_rollover(f, skb, idx, true, num);

        po = pkt_sk(rcu_dereference(f->arr[idx]));
        return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev);
}

DEFINE_MUTEX(fanout_mutex);
EXPORT_SYMBOL_GPL(fanout_mutex);
static LIST_HEAD(fanout_list);
static u16 fanout_next_id;

static void __fanout_link(struct sock *sk, struct packet_sock *po)
{
        struct packet_fanout *f = po->fanout;

        spin_lock(&f->lock);
        rcu_assign_pointer(f->arr[f->num_members], sk);
        smp_wmb();
        f->num_members++;
        if (f->num_members == 1)
                dev_add_pack(&f->prot_hook);
        spin_unlock(&f->lock);
}

static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
{
        struct packet_fanout *f = po->fanout;
        int i;

        spin_lock(&f->lock);
        for (i = 0; i < f->num_members; i++) {
                if (rcu_dereference_protected(f->arr[i],
                                              lockdep_is_held(&f->lock)) == sk)
                        break;
        }
        BUG_ON(i >= f->num_members);
        rcu_assign_pointer(f->arr[i],
                           rcu_dereference_protected(f->arr[f->num_members - 1],
                                                     lockdep_is_held(&f->lock)));
        f->num_members--;
        if (f->num_members == 0)
                __dev_remove_pack(&f->prot_hook);
        spin_unlock(&f->lock);
}

static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
{
        if (sk->sk_family != PF_PACKET)
                return false;

        return ptype->af_packet_priv == pkt_sk(sk)->fanout;
}

static void fanout_init_data(struct packet_fanout *f)
{
        switch (f->type) {
        case PACKET_FANOUT_LB:
                atomic_set(&f->rr_cur, 0);
                break;
        case PACKET_FANOUT_CBPF:
        case PACKET_FANOUT_EBPF:
                RCU_INIT_POINTER(f->bpf_prog, NULL);
                break;
        }
}

static void __fanout_set_data_bpf(struct packet_fanout *f, struct bpf_prog *new)
{
        struct bpf_prog *old;

        spin_lock(&f->lock);
        old = rcu_dereference_protected(f->bpf_prog, lockdep_is_held(&f->lock));
        rcu_assign_pointer(f->bpf_prog, new);
        spin_unlock(&f->lock);

        if (old) {
                synchronize_net();
                bpf_prog_destroy(old);
        }
}

static int fanout_set_data_cbpf(struct packet_sock *po, sockptr_t data,
                                unsigned int len)
{
        struct bpf_prog *new;
        struct sock_fprog fprog;
        int ret;

        if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
                return -EPERM;

        ret = copy_bpf_fprog_from_user(&fprog, data, len);
        if (ret)
                return ret;

        ret = bpf_prog_create_from_user(&new, &fprog, NULL, false);
        if (ret)
                return ret;

        __fanout_set_data_bpf(po->fanout, new);
        return 0;
}

static int fanout_set_data_ebpf(struct packet_sock *po, sockptr_t data,
                                unsigned int len)
{
        struct bpf_prog *new;
        u32 fd;

        if (sock_flag(&po->sk, SOCK_FILTER_LOCKED))
                return -EPERM;
        if (len != sizeof(fd))
                return -EINVAL;
        if (copy_from_sockptr(&fd, data, len))
                return -EFAULT;

        new = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
        if (IS_ERR(new))
                return PTR_ERR(new);

        __fanout_set_data_bpf(po->fanout, new);
        return 0;
}

static int fanout_set_data(struct packet_sock *po, sockptr_t data,
                           unsigned int len)
{
        switch (po->fanout->type) {
        case PACKET_FANOUT_CBPF:
                return fanout_set_data_cbpf(po, data, len);
        case PACKET_FANOUT_EBPF:
                return fanout_set_data_ebpf(po, data, len);
        default:
                return -EINVAL;
        }
}

static void fanout_release_data(struct packet_fanout *f)
{
        switch (f->type) {
        case PACKET_FANOUT_CBPF:
        case PACKET_FANOUT_EBPF:
                __fanout_set_data_bpf(f, NULL);
        }
}

static bool __fanout_id_is_free(struct sock *sk, u16 candidate_id)
{
        struct packet_fanout *f;

        list_for_each_entry(f, &fanout_list, list) {
                if (f->id == candidate_id &&
                    read_pnet(&f->net) == sock_net(sk)) {
                        return false;
                }
        }
        return true;
}

static bool fanout_find_new_id(struct sock *sk, u16 *new_id)
{
        u16 id = fanout_next_id;

        do {
                if (__fanout_id_is_free(sk, id)) {
                        *new_id = id;
                        fanout_next_id = id + 1;
                        return true;
                }

                id++;
        } while (id != fanout_next_id);

        return false;
}

static int fanout_add(struct sock *sk, struct fanout_args *args)
{
        struct packet_rollover *rollover = NULL;
        struct packet_sock *po = pkt_sk(sk);
        u16 type_flags = args->type_flags;
        struct packet_fanout *f, *match;
        u8 type = type_flags & 0xff;
        u8 flags = type_flags >> 8;
        u16 id = args->id;
        int err;

        switch (type) {
        case PACKET_FANOUT_ROLLOVER:
                if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)
                        return -EINVAL;
                break;
        case PACKET_FANOUT_HASH:
        case PACKET_FANOUT_LB:
        case PACKET_FANOUT_CPU:
        case PACKET_FANOUT_RND:
        case PACKET_FANOUT_QM:
        case PACKET_FANOUT_CBPF:
        case PACKET_FANOUT_EBPF:
                break;
        default:
                return -EINVAL;
        }

        mutex_lock(&fanout_mutex);

        err = -EALREADY;
        if (po->fanout)
                goto out;

        if (type == PACKET_FANOUT_ROLLOVER ||
            (type_flags & PACKET_FANOUT_FLAG_ROLLOVER)) {
                err = -ENOMEM;
                rollover = kzalloc(sizeof(*rollover), GFP_KERNEL);
                if (!rollover)
                        goto out;
                atomic_long_set(&rollover->num, 0);
                atomic_long_set(&rollover->num_huge, 0);
                atomic_long_set(&rollover->num_failed, 0);
        }

        if (type_flags & PACKET_FANOUT_FLAG_UNIQUEID) {
                if (id != 0) {
                        err = -EINVAL;
                        goto out;
                }
                if (!fanout_find_new_id(sk, &id)) {
                        err = -ENOMEM;
                        goto out;
                }
                /* ephemeral flag for the first socket in the group: drop it */
                flags &= ~(PACKET_FANOUT_FLAG_UNIQUEID >> 8);
        }

        match = NULL;
        list_for_each_entry(f, &fanout_list, list) {
                if (f->id == id &&
                    read_pnet(&f->net) == sock_net(sk)) {
                        match = f;
                        break;
                }
        }
        err = -EINVAL;
        if (match) {
                if (match->flags != flags)
                        goto out;
                if (args->max_num_members &&
                    args->max_num_members != match->max_num_members)
                        goto out;
        } else {
                if (args->max_num_members > PACKET_FANOUT_MAX)
                        goto out;
                if (!args->max_num_members)
                        /* legacy PACKET_FANOUT_MAX */
                        args->max_num_members = 256;
                err = -ENOMEM;
                match = kvzalloc(struct_size(match, arr, args->max_num_members),
                                 GFP_KERNEL);
                if (!match)
                        goto out;
                write_pnet(&match->net, sock_net(sk));
                match->id = id;
                match->type = type;
                match->flags = flags;
                INIT_LIST_HEAD(&match->list);
                spin_lock_init(&match->lock);
                refcount_set(&match->sk_ref, 0);
                fanout_init_data(match);
                match->prot_hook.type = po->prot_hook.type;
                match->prot_hook.dev = po->prot_hook.dev;
                match->prot_hook.func = packet_rcv_fanout;
                match->prot_hook.af_packet_priv = match;
                match->prot_hook.af_packet_net = read_pnet(&match->net);
                match->prot_hook.id_match = match_fanout_group;
                match->max_num_members = args->max_num_members;
                match->prot_hook.ignore_outgoing = type_flags & PACKET_FANOUT_FLAG_IGNORE_OUTGOING;
                list_add(&match->list, &fanout_list);
        }
        err = -EINVAL;

        spin_lock(&po->bind_lock);
        if (po->num &&
            match->type == type &&
            match->prot_hook.type == po->prot_hook.type &&
            match->prot_hook.dev == po->prot_hook.dev) {
                err = -ENOSPC;
                if (refcount_read(&match->sk_ref) < match->max_num_members) {
                        /* Paired with packet_setsockopt(PACKET_FANOUT_DATA) */
                        WRITE_ONCE(po->fanout, match);

                        po->rollover = rollover;
                        rollover = NULL;
                        refcount_set(&match->sk_ref, refcount_read(&match->sk_ref) + 1);
                        if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
                                __dev_remove_pack(&po->prot_hook);
                                __fanout_link(sk, po);
                        }
                        err = 0;
                }
        }
        spin_unlock(&po->bind_lock);

        if (err && !refcount_read(&match->sk_ref)) {
                list_del(&match->list);
                kvfree(match);
        }

out:
        kfree(rollover);
        mutex_unlock(&fanout_mutex);
        return err;
}

/* If pkt_sk(sk)->fanout->sk_ref is zero, this function removes
 * pkt_sk(sk)->fanout from fanout_list and returns pkt_sk(sk)->fanout.
 * It is the responsibility of the caller to call fanout_release_data() and
 * free the returned packet_fanout (after synchronize_net())
 */
static struct packet_fanout *fanout_release(struct sock *sk)
{
        struct packet_sock *po = pkt_sk(sk);
        struct packet_fanout *f;

        mutex_lock(&fanout_mutex);
        f = po->fanout;
        if (f) {
                po->fanout = NULL;

                if (refcount_dec_and_test(&f->sk_ref))
                        list_del(&f->list);
                else
                        f = NULL;
        }
        mutex_unlock(&fanout_mutex);

        return f;
}

static bool packet_extra_vlan_len_allowed(const struct net_device *dev,
                                          struct sk_buff *skb)
{
        /* Earlier code assumed this would be a VLAN pkt, double-check
         * this now that we have the actual packet in hand. We can only
         * do this check on Ethernet devices.
         */
        if (unlikely(dev->type != ARPHRD_ETHER))
                return false;

        skb_reset_mac_header(skb);
        return likely(eth_hdr(skb)->h_proto == htons(ETH_P_8021Q));
}

static const struct proto_ops packet_ops;

static const struct proto_ops packet_ops_spkt;

static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
                           struct packet_type *pt, struct net_device *orig_dev)
{
        struct sock *sk;
        struct sockaddr_pkt *spkt;

        /*
         *        When we registered the protocol we saved the socket in the data
         *        field for just this event.
         */

        sk = pt->af_packet_priv;

        /*
         *        Yank back the headers [hope the device set this
         *        right or kerboom...]
         *
         *        Incoming packets have ll header pulled,
         *        push it back.
         *
         *        For outgoing ones skb->data == skb_mac_header(skb)
         *        so that this procedure is noop.
         */

        if (skb->pkt_type == PACKET_LOOPBACK)
                goto out;

        if (!net_eq(dev_net(dev), sock_net(sk)))
                goto out;

        skb = skb_share_check(skb, GFP_ATOMIC);
        if (skb == NULL)
                goto oom;

        /* drop any routing info */
        skb_dst_drop(skb);

        /* drop conntrack reference */
        nf_reset_ct(skb);

        spkt = &PACKET_SKB_CB(skb)->sa.pkt;

        skb_push(skb, skb->data - skb_mac_header(skb));

        /*
         *        The SOCK_PACKET socket receives _all_ frames.
         */

        spkt->spkt_family = dev->type;
        strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
        spkt->spkt_protocol = skb->protocol;

        /*
         *        Charge the memory to the socket. This is done specifically
         *        to prevent sockets using all the memory up.
         */

        if (sock_queue_rcv_skb(sk, skb) == 0)
                return 0;

out:
        kfree_skb(skb);
oom:
        return 0;
}

static void packet_parse_headers(struct sk_buff *skb, struct socket *sock)
{
        int depth;

        if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) &&
            sock->type == SOCK_RAW) {
                skb_reset_mac_header(skb);
                skb->protocol = dev_parse_header_protocol(skb);
        }

        /* Move network header to the right position for VLAN tagged packets */
        if (likely(skb->dev->type == ARPHRD_ETHER) &&
            eth_type_vlan(skb->protocol) &&
            vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0)
                skb_set_network_header(skb, depth);

        skb_probe_transport_header(skb);
}

/*
 *        Output a raw packet to a device layer. This bypasses all the other
 *        protocol layers and you must therefore supply it with a complete frame
 */

static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg,
                               size_t len)
{
        struct sock *sk = sock->sk;
        DECLARE_SOCKADDR(struct sockaddr_pkt *, saddr, msg->msg_name);
        struct sk_buff *skb = NULL;
        struct net_device *dev;
        struct sockcm_cookie sockc;
        __be16 proto = 0;
        int err;
        int extra_len = 0;

        /*
         *        Get and verify the address.
         */

        if (saddr) {
                if (msg->msg_namelen < sizeof(struct sockaddr))
                        return -EINVAL;
                if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
                        proto = saddr->spkt_protocol;
        } else
                return -ENOTCONN;        /* SOCK_PACKET must be sent giving an address */

        /*
         *        Find the device first to size check it
         */

        saddr->spkt_device[sizeof(saddr->spkt_device) - 1] = 0;
retry:
        rcu_read_lock();
        dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
        err = -ENODEV;
        if (dev == NULL)
                goto out_unlock;

        err = -ENETDOWN;
        if (!(dev->flags & IFF_UP))
                goto out_unlock;

        /*
         * You may not queue a frame bigger than the mtu. This is the lowest level
         * raw protocol and you must do your own fragmentation at this level.
         */

        if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
                if (!netif_supports_nofcs(dev)) {
                        err = -EPROTONOSUPPORT;
                        goto out_unlock;
                }
                extra_len = 4; /* We're doing our own CRC */
        }

        err = -EMSGSIZE;
        if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len)
                goto out_unlock;

        if (!skb) {
                size_t reserved = LL_RESERVED_SPACE(dev);
                int tlen = dev->needed_tailroom;
                unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;

                rcu_read_unlock();
                skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL);
                if (skb == NULL)
                        return -ENOBUFS;
                /* FIXME: Save some space for broken drivers that write a hard
                 * header at transmission time by themselves. PPP is the notable
                 * one here. This should really be fixed at the driver level.
                 */
                skb_reserve(skb, reserved);
                skb_reset_network_header(skb);

                /* Try to align data part correctly */
                if (hhlen) {
                        skb->data -= hhlen;
                        skb->tail -= hhlen;
                        if (len < hhlen)
                                skb_reset_network_header(skb);
                }
                err = memcpy_from_msg(skb_put(skb, len), msg, len);
                if (err)
                        goto out_free;
                goto retry;
        }

        if (!dev_validate_header(dev, skb->data, len) || !skb->len) {
                err = -EINVAL;
                goto out_unlock;
        }
        if (len > (dev->mtu + dev->hard_header_len + extra_len) &&
            !packet_extra_vlan_len_allowed(dev, skb)) {
                err = -EMSGSIZE;
                goto out_unlock;
        }

        sockcm_init(&sockc, sk);
        if (msg->msg_controllen) {
                err = sock_cmsg_send(sk, msg, &sockc);
                if (unlikely(err))
                        goto out_unlock;
        }

        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sockc.priority;
        skb->mark = sockc.mark;
        skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid);
        skb_setup_tx_timestamp(skb, &sockc);

        if (unlikely(extra_len == 4))
                skb->no_fcs = 1;

        packet_parse_headers(skb, sock);

        dev_queue_xmit(skb);
        rcu_read_unlock();
        return len;

out_unlock:
        rcu_read_unlock();
out_free:
        kfree_skb(skb);
        return err;
}

static unsigned int run_filter(struct sk_buff *skb,
                               const struct sock *sk,
                               unsigned int res)
{
        struct sk_filter *filter;

        rcu_read_lock();
        filter = rcu_dereference(sk->sk_filter);
        if (filter != NULL)
                res = bpf_prog_run_clear_cb(filter->prog, skb);
        rcu_read_unlock();

        return res;
}

static int packet_rcv_vnet(struct msghdr *msg, const struct sk_buff *skb,
                           size_t *len, int vnet_hdr_sz)
{
        struct virtio_net_hdr_mrg_rxbuf vnet_hdr = { .num_buffers = 0 };

        if (*len < vnet_hdr_sz)
                return -EINVAL;
        *len -= vnet_hdr_sz;

        if (virtio_net_hdr_from_skb(skb, (struct virtio_net_hdr *)&vnet_hdr, vio_le(), true, 0))
                return -EINVAL;

        return memcpy_to_msg(msg, (void *)&vnet_hdr, vnet_hdr_sz);
}

/*
 * This function makes lazy skb cloning in hope that most of packets
 * are discarded by BPF.
 *
 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
 * and skb->cb are mangled. It works because (and until) packets
 * falling here are owned by current CPU. Output packets are cloned
 * by dev_queue_xmit_nit(), input packets are processed by net_bh
 * sequentially, so that if we return skb to original state on exit,
 * we will not harm anyone.
 */

static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
                      struct packet_type *pt, struct net_device *orig_dev)
{
        enum skb_drop_reason drop_reason = SKB_CONSUMED;
        struct sock *sk = NULL;
        struct sockaddr_ll *sll;
        struct packet_sock *po;
        u8 *skb_head = skb->data;
        int skb_len = skb->len;
        unsigned int snaplen, res;

        if (skb->pkt_type == PACKET_LOOPBACK)
                goto drop;

        sk = pt->af_packet_priv;
        po = pkt_sk(sk);

        if (!net_eq(dev_net(dev), sock_net(sk)))
                goto drop;

        skb->dev = dev;

        if (dev_has_header(dev)) {
                /* The device has an explicit notion of ll header,
                 * exported to higher levels.
                 *
                 * Otherwise, the device hides details of its frame
                 * structure, so that corresponding packet head is
                 * never delivered to user.
                 */
                if (sk->sk_type != SOCK_DGRAM)
                        skb_push(skb, skb->data - skb_mac_header(skb));
                else if (skb->pkt_type == PACKET_OUTGOING) {
                        /* Special case: outgoing packets have ll header at head */
                        skb_pull(skb, skb_network_offset(skb));
                }
        }

        snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);

        res = run_filter(skb, sk, snaplen);
        if (!res)
                goto drop_n_restore;
        if (snaplen > res)
                snaplen = res;

        if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
                goto drop_n_acct;

        if (skb_shared(skb)) {
                struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
                if (nskb == NULL)
                        goto drop_n_acct;

                if (skb_head != skb->data) {
                        skb->data = skb_head;
                        skb->len = skb_len;
                }
                consume_skb(skb);
                skb = nskb;
        }

        sock_skb_cb_check_size(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8);

        sll = &PACKET_SKB_CB(skb)->sa.ll;
        sll->sll_hatype = dev->type;
        sll->sll_pkttype = skb->pkt_type;
        if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
                sll->sll_ifindex = orig_dev->ifindex;
        else
                sll->sll_ifindex = dev->ifindex;

        sll->sll_halen = dev_parse_header(skb, sll->sll_addr);

        /* sll->sll_family and sll->sll_protocol are set in packet_recvmsg().
         * Use their space for storing the original skb length.
         */
        PACKET_SKB_CB(skb)->sa.origlen = skb->len;

        if (pskb_trim(skb, snaplen))
                goto drop_n_acct;

        skb_set_owner_r(skb, sk);
        skb->dev = NULL;
        skb_dst_drop(skb);

        /* drop conntrack reference */
        nf_reset_ct(skb);

        spin_lock(&sk->sk_receive_queue.lock);
        po->stats.stats1.tp_packets++;
        sock_skb_set_dropcount(sk, skb);
        skb_clear_delivery_time(skb);
        __skb_queue_tail(&sk->sk_receive_queue, skb);
        spin_unlock(&sk->sk_receive_queue.lock);
        sk->sk_data_ready(sk);
        return 0;

drop_n_acct:
        atomic_inc(&po->tp_drops);
        atomic_inc(&sk->sk_drops);
        drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;

drop_n_restore:
        if (skb_head != skb->data && skb_shared(skb)) {
                skb->data = skb_head;
                skb->len = skb_len;
        }
drop:
        sk_skb_reason_drop(sk, skb, drop_reason);
        return 0;
}

static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
                       struct packet_type *pt, struct net_device *orig_dev)
{
        enum skb_drop_reason drop_reason = SKB_CONSUMED;
        struct sock *sk = NULL;
        struct packet_sock *po;
        struct sockaddr_ll *sll;
        union tpacket_uhdr h;
        u8 *skb_head = skb->data;
        int skb_len = skb->len;
        unsigned int snaplen, res;
        unsigned long status = TP_STATUS_USER;
        unsigned short macoff, hdrlen;
        unsigned int netoff;
        struct sk_buff *copy_skb = NULL;
        struct timespec64 ts;
        __u32 ts_status;
        unsigned int slot_id = 0;
        int vnet_hdr_sz = 0;

        /* struct tpacket{2,3}_hdr is aligned to a multiple of TPACKET_ALIGNMENT.
         * We may add members to them until current aligned size without forcing
         * userspace to call getsockopt(..., PACKET_HDRLEN, ...).
         */
        BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h2)) != 32);
        BUILD_BUG_ON(TPACKET_ALIGN(sizeof(*h.h3)) != 48);

        if (skb->pkt_type == PACKET_LOOPBACK)
                goto drop;

        sk = pt->af_packet_priv;
        po = pkt_sk(sk);

        if (!net_eq(dev_net(dev), sock_net(sk)))
                goto drop;

        if (dev_has_header(dev)) {
                if (sk->sk_type != SOCK_DGRAM)
                        skb_push(skb, skb->data - skb_mac_header(skb));
                else if (skb->pkt_type == PACKET_OUTGOING) {
                        /* Special case: outgoing packets have ll header at head */
                        skb_pull(skb, skb_network_offset(skb));
                }
        }

        snaplen = skb_frags_readable(skb) ? skb->len : skb_headlen(skb);

        res = run_filter(skb, sk, snaplen);
        if (!res)
                goto drop_n_restore;

        /* If we are flooded, just give up */
        if (__packet_rcv_has_room(po, skb) == ROOM_NONE) {
                atomic_inc(&po->tp_drops);
                goto drop_n_restore;
        }

        if (skb->ip_summed == CHECKSUM_PARTIAL)
                status |= TP_STATUS_CSUMNOTREADY;
        else if (skb->pkt_type != PACKET_OUTGOING &&
                 skb_csum_unnecessary(skb))
                status |= TP_STATUS_CSUM_VALID;
        if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
                status |= TP_STATUS_GSO_TCP;

        if (snaplen > res)
                snaplen = res;

        if (sk->sk_type == SOCK_DGRAM) {
                macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
                                  po->tp_reserve;
        } else {
                unsigned int maclen = skb_network_offset(skb);
                netoff = TPACKET_ALIGN(po->tp_hdrlen +
                                       (maclen < 16 ? 16 : maclen)) +
                                       po->tp_reserve;
                vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
                if (vnet_hdr_sz)
                        netoff += vnet_hdr_sz;
                macoff = netoff - maclen;
        }
        if (netoff > USHRT_MAX) {
                atomic_inc(&po->tp_drops);
                goto drop_n_restore;
        }
        if (po->tp_version <= TPACKET_V2) {
                if (macoff + snaplen > po->rx_ring.frame_size) {
                        if (READ_ONCE(po->copy_thresh) &&
                            atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) {
                                if (skb_shared(skb)) {
                                        copy_skb = skb_clone(skb, GFP_ATOMIC);
                                } else {
                                        copy_skb = skb_get(skb);
                                        skb_head = skb->data;
                                }
                                if (copy_skb) {
                                        memset(&PACKET_SKB_CB(copy_skb)->sa.ll, 0,
                                               sizeof(PACKET_SKB_CB(copy_skb)->sa.ll));
                                        skb_set_owner_r(copy_skb, sk);
                                }
                        }
                        snaplen = po->rx_ring.frame_size - macoff;
                        if ((int)snaplen < 0) {
                                snaplen = 0;
                                vnet_hdr_sz = 0;
                        }
                }
        } else if (unlikely(macoff + snaplen >
                            GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len)) {
                u32 nval;

                nval = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len - macoff;
                pr_err_once("tpacket_rcv: packet too big, clamped from %u to %u. macoff=%u\n",
                            snaplen, nval, macoff);
                snaplen = nval;
                if (unlikely((int)snaplen < 0)) {
                        snaplen = 0;
                        macoff = GET_PBDQC_FROM_RB(&po->rx_ring)->max_frame_len;
                        vnet_hdr_sz = 0;
                }
        }
        spin_lock(&sk->sk_receive_queue.lock);
        h.raw = packet_current_rx_frame(po, skb,
                                        TP_STATUS_KERNEL, (macoff+snaplen));
        if (!h.raw)
                goto drop_n_account;

        if (po->tp_version <= TPACKET_V2) {
                slot_id = po->rx_ring.head;
                if (test_bit(slot_id, po->rx_ring.rx_owner_map))
                        goto drop_n_account;
                __set_bit(slot_id, po->rx_ring.rx_owner_map);
        }

        if (vnet_hdr_sz &&
            virtio_net_hdr_from_skb(skb, h.raw + macoff -
                                    sizeof(struct virtio_net_hdr),
                                    vio_le(), true, 0)) {
                if (po->tp_version == TPACKET_V3)
                        prb_clear_blk_fill_status(&po->rx_ring);
                goto drop_n_account;
        }

        if (po->tp_version <= TPACKET_V2) {
                packet_increment_rx_head(po, &po->rx_ring);
        /*
         * LOSING will be reported till you read the stats,
         * because it's COR - Clear On Read.
         * Anyways, moving it for V1/V2 only as V3 doesn't need this
         * at packet level.
         */
                if (atomic_read(&po->tp_drops))
                        status |= TP_STATUS_LOSING;
        }

        po->stats.stats1.tp_packets++;
        if (copy_skb) {
                status |= TP_STATUS_COPY;
                skb_clear_delivery_time(copy_skb);
                __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
        }
        spin_unlock(&sk->sk_receive_queue.lock);

        skb_copy_bits(skb, 0, h.raw + macoff, snaplen);

        /* Always timestamp; prefer an existing software timestamp taken
         * closer to the time of capture.
         */
        ts_status = tpacket_get_timestamp(skb, &ts,
                                          READ_ONCE(po->tp_tstamp) |
                                          SOF_TIMESTAMPING_SOFTWARE);
        if (!ts_status)
                ktime_get_real_ts64(&ts);

        status |= ts_status;

        switch (po->tp_version) {
        case TPACKET_V1:
                h.h1->tp_len = skb->len;
                h.h1->tp_snaplen = snaplen;
                h.h1->tp_mac = macoff;
                h.h1->tp_net = netoff;
                h.h1->tp_sec = ts.tv_sec;
                h.h1->tp_usec = ts.tv_nsec / NSEC_PER_USEC;
                hdrlen = sizeof(*h.h1);
                break;
        case TPACKET_V2:
                h.h2->tp_len = skb->len;
                h.h2->tp_snaplen = snaplen;
                h.h2->tp_mac = macoff;
                h.h2->tp_net = netoff;
                h.h2->tp_sec = ts.tv_sec;
                h.h2->tp_nsec = ts.tv_nsec;
                if (skb_vlan_tag_present(skb)) {
                        h.h2->tp_vlan_tci = skb_vlan_tag_get(skb);
                        h.h2->tp_vlan_tpid = ntohs(skb->vlan_proto);
                        status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
                } else if (unlikely(sk->sk_type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
                        h.h2->tp_vlan_tci = vlan_get_tci(skb, skb->dev);
                        h.h2->tp_vlan_tpid = ntohs(skb->protocol);
                        status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
                } else {
                        h.h2->tp_vlan_tci = 0;
                        h.h2->tp_vlan_tpid = 0;
                }
                memset(h.h2->tp_padding, 0, sizeof(h.h2->tp_padding));
                hdrlen = sizeof(*h.h2);
                break;
        case TPACKET_V3:
                /* tp_nxt_offset,vlan are already populated above.
                 * So DONT clear those fields here
                 */
                h.h3->tp_status |= status;
                h.h3->tp_len = skb->len;
                h.h3->tp_snaplen = snaplen;
                h.h3->tp_mac = macoff;
                h.h3->tp_net = netoff;
                h.h3->tp_sec  = ts.tv_sec;
                h.h3->tp_nsec = ts.tv_nsec;
                memset(h.h3->tp_padding, 0, sizeof(h.h3->tp_padding));
                hdrlen = sizeof(*h.h3);
                break;
        default:
                BUG();
        }

        sll = h.raw + TPACKET_ALIGN(hdrlen);
        sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
        sll->sll_family = AF_PACKET;
        sll->sll_hatype = dev->type;
        sll->sll_protocol = (sk->sk_type == SOCK_DGRAM) ?
                vlan_get_protocol_dgram(skb) : skb->protocol;
        sll->sll_pkttype = skb->pkt_type;
        if (unlikely(packet_sock_flag(po, PACKET_SOCK_ORIGDEV)))
                sll->sll_ifindex = orig_dev->ifindex;
        else
                sll->sll_ifindex = dev->ifindex;

        smp_mb();

#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
        if (po->tp_version <= TPACKET_V2) {
                u8 *start, *end;

                end = (u8 *) PAGE_ALIGN((unsigned long) h.raw +
                                        macoff + snaplen);

                for (start = h.raw; start < end; start += PAGE_SIZE)
                        flush_dcache_page(pgv_to_page(start));
        }
        smp_wmb();
#endif

        if (po->tp_version <= TPACKET_V2) {
                spin_lock(&sk->sk_receive_queue.lock);
                __packet_set_status(po, h.raw, status);
                __clear_bit(slot_id, po->rx_ring.rx_owner_map);
                spin_unlock(&sk->sk_receive_queue.lock);
                sk->sk_data_ready(sk);
        } else if (po->tp_version == TPACKET_V3) {
                prb_clear_blk_fill_status(&po->rx_ring);
        }

drop_n_restore:
        if (skb_head != skb->data && skb_shared(skb)) {
                skb->data = skb_head;
                skb->len = skb_len;
        }
drop:
        sk_skb_reason_drop(sk, skb, drop_reason);
        return 0;

drop_n_account:
        spin_unlock(&sk->sk_receive_queue.lock);
        atomic_inc(&po->tp_drops);
        drop_reason = SKB_DROP_REASON_PACKET_SOCK_ERROR;

        sk->sk_data_ready(sk);
        sk_skb_reason_drop(sk, copy_skb, drop_reason);
        goto drop_n_restore;
}

static void tpacket_destruct_skb(struct sk_buff *skb)
{
        struct packet_sock *po = pkt_sk(skb->sk);

        if (likely(po->tx_ring.pg_vec)) {
                void *ph;
                __u32 ts;

                ph = skb_zcopy_get_nouarg(skb);
                packet_dec_pending(&po->tx_ring);

                ts = __packet_set_timestamp(po, ph, skb);
                __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts);

                complete(&po->skb_completion);
        }

        sock_wfree(skb);
}

static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len)
{
        if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
            (__virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
             __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2 >
              __virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len)))
                vnet_hdr->hdr_len = __cpu_to_virtio16(vio_le(),
                         __virtio16_to_cpu(vio_le(), vnet_hdr->csum_start) +
                        __virtio16_to_cpu(vio_le(), vnet_hdr->csum_offset) + 2);

        if (__virtio16_to_cpu(vio_le(), vnet_hdr->hdr_len) > len)
                return -EINVAL;

        return 0;
}

static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len,
                                 struct virtio_net_hdr *vnet_hdr, int vnet_hdr_sz)
{
        int ret;

        if (*len < vnet_hdr_sz)
                return -EINVAL;
        *len -= vnet_hdr_sz;

        if (!copy_from_iter_full(vnet_hdr, sizeof(*vnet_hdr), &msg->msg_iter))
                return -EFAULT;

        ret = __packet_snd_vnet_parse(vnet_hdr, *len);
        if (ret)
                return ret;

        /* move iter to point to the start of mac header */
        if (vnet_hdr_sz != sizeof(struct virtio_net_hdr))
                iov_iter_advance(&msg->msg_iter, vnet_hdr_sz - sizeof(struct virtio_net_hdr));

        return 0;
}

static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
                void *frame, struct net_device *dev, void *data, int tp_len,
                __be16 proto, unsigned char *addr, int hlen, int copylen,
                const struct sockcm_cookie *sockc)
{
        union tpacket_uhdr ph;
        int to_write, offset, len, nr_frags, len_max;
        struct socket *sock = po->sk.sk_socket;
        struct page *page;
        int err;

        ph.raw = frame;

        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sockc->priority;
        skb->mark = sockc->mark;
        skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid);
        skb_setup_tx_timestamp(skb, sockc);
        skb_zcopy_set_nouarg(skb, ph.raw);

        skb_reserve(skb, hlen);
        skb_reset_network_header(skb);

        to_write = tp_len;

        if (sock->type == SOCK_DGRAM) {
                err = dev_hard_header(skb, dev, ntohs(proto), addr,
                                NULL, tp_len);
                if (unlikely(err < 0))
                        return -EINVAL;
        } else if (copylen) {
                int hdrlen = min_t(int, copylen, tp_len);

                skb_push(skb, dev->hard_header_len);
                skb_put(skb, copylen - dev->hard_header_len);
                err = skb_store_bits(skb, 0, data, hdrlen);
                if (unlikely(err))
                        return err;
                if (!dev_validate_header(dev, skb->data, hdrlen))
                        return -EINVAL;

                data += hdrlen;
                to_write -= hdrlen;
        }

        offset = offset_in_page(data);
        len_max = PAGE_SIZE - offset;
        len = ((to_write > len_max) ? len_max : to_write);

        skb->data_len = to_write;
        skb->len += to_write;
        skb->truesize += to_write;
        refcount_add(to_write, &po->sk.sk_wmem_alloc);

        while (likely(to_write)) {
                nr_frags = skb_shinfo(skb)->nr_frags;

                if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
                        pr_err("Packet exceed the number of skb frags(%u)\n",
                               (unsigned int)MAX_SKB_FRAGS);
                        return -EFAULT;
                }

                page = pgv_to_page(data);
                data += len;
                flush_dcache_page(page);
                get_page(page);
                skb_fill_page_desc(skb, nr_frags, page, offset, len);
                to_write -= len;
                offset = 0;
                len_max = PAGE_SIZE;
                len = ((to_write > len_max) ? len_max : to_write);
        }

        packet_parse_headers(skb, sock);

        return tp_len;
}

static int tpacket_parse_header(struct packet_sock *po, void *frame,
                                int size_max, void **data)
{
        union tpacket_uhdr ph;
        int tp_len, off;

        ph.raw = frame;

        switch (po->tp_version) {
        case TPACKET_V3:
                if (ph.h3->tp_next_offset != 0) {
                        pr_warn_once("variable sized slot not supported");
                        return -EINVAL;
                }
                tp_len = ph.h3->tp_len;
                break;
        case TPACKET_V2:
                tp_len = ph.h2->tp_len;
                break;
        default:
                tp_len = ph.h1->tp_len;
                break;
        }
        if (unlikely(tp_len > size_max)) {
                pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
                return -EMSGSIZE;
        }

        if (unlikely(packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF))) {
                int off_min, off_max;

                off_min = po->tp_hdrlen - sizeof(struct sockaddr_ll);
                off_max = po->tx_ring.frame_size - tp_len;
                if (po->sk.sk_type == SOCK_DGRAM) {
                        switch (po->tp_version) {
                        case TPACKET_V3:
                                off = ph.h3->tp_net;
                                break;
                        case TPACKET_V2:
                                off = ph.h2->tp_net;
                                break;
                        default:
                                off = ph.h1->tp_net;
                                break;
                        }
                } else {
                        switch (po->tp_version) {
                        case TPACKET_V3:
                                off = ph.h3->tp_mac;
                                break;
                        case TPACKET_V2:
                                off = ph.h2->tp_mac;
                                break;
                        default:
                                off = ph.h1->tp_mac;
                                break;
                        }
                }
                if (unlikely((off < off_min) || (off_max < off)))
                        return -EINVAL;
        } else {
                off = po->tp_hdrlen - sizeof(struct sockaddr_ll);
        }

        *data = frame + off;
        return tp_len;
}

static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
{
        struct sk_buff *skb = NULL;
        struct net_device *dev;
        struct virtio_net_hdr *vnet_hdr = NULL;
        struct sockcm_cookie sockc;
        __be16 proto;
        int err, reserve = 0;
        void *ph;
        DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
        bool need_wait = !(msg->msg_flags & MSG_DONTWAIT);
        int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
        unsigned char *addr = NULL;
        int tp_len, size_max;
        void *data;
        int len_sum = 0;
        int status = TP_STATUS_AVAILABLE;
        int hlen, tlen, copylen = 0;
        long timeo = 0;

        mutex_lock(&po->pg_vec_lock);

        /* packet_sendmsg() check on tx_ring.pg_vec was lockless,
         * we need to confirm it under protection of pg_vec_lock.
         */
        if (unlikely(!po->tx_ring.pg_vec)) {
                err = -EBUSY;
                goto out;
        }
        if (likely(saddr == NULL)) {
                dev        = packet_cached_dev_get(po);
                proto        = READ_ONCE(po->num);
        } else {
                err = -EINVAL;
                if (msg->msg_namelen < sizeof(struct sockaddr_ll))
                        goto out;
                if (msg->msg_namelen < (saddr->sll_halen
                                        + offsetof(struct sockaddr_ll,
                                                sll_addr)))
                        goto out;
                proto        = saddr->sll_protocol;
                dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex);
                if (po->sk.sk_socket->type == SOCK_DGRAM) {
                        if (dev && msg->msg_namelen < dev->addr_len +
                                   offsetof(struct sockaddr_ll, sll_addr))
                                goto out_put;
                        addr = saddr->sll_addr;
                }
        }

        err = -ENXIO;
        if (unlikely(dev == NULL))
                goto out;
        err = -ENETDOWN;
        if (unlikely(!(dev->flags & IFF_UP)))
                goto out_put;

        sockcm_init(&sockc, &po->sk);
        if (msg->msg_controllen) {
                err = sock_cmsg_send(&po->sk, msg, &sockc);
                if (unlikely(err))
                        goto out_put;
        }

        if (po->sk.sk_socket->type == SOCK_RAW)
                reserve = dev->hard_header_len;
        size_max = po->tx_ring.frame_size
                - (po->tp_hdrlen - sizeof(struct sockaddr_ll));

        if ((size_max > dev->mtu + reserve + VLAN_HLEN) && !vnet_hdr_sz)
                size_max = dev->mtu + reserve + VLAN_HLEN;

        reinit_completion(&po->skb_completion);

        do {
                ph = packet_current_frame(po, &po->tx_ring,
                                          TP_STATUS_SEND_REQUEST);
                if (unlikely(ph == NULL)) {
                        if (need_wait && skb) {
                                timeo = sock_sndtimeo(&po->sk, msg->msg_flags & MSG_DONTWAIT);
                                timeo = wait_for_completion_interruptible_timeout(&po->skb_completion, timeo);
                                if (timeo <= 0) {
                                        err = !timeo ? -ETIMEDOUT : -ERESTARTSYS;
                                        goto out_put;
                                }
                        }
                        /* check for additional frames */
                        continue;
                }

                skb = NULL;
                tp_len = tpacket_parse_header(po, ph, size_max, &data);
                if (tp_len < 0)
                        goto tpacket_error;

                status = TP_STATUS_SEND_REQUEST;
                hlen = LL_RESERVED_SPACE(dev);
                tlen = dev->needed_tailroom;
                if (vnet_hdr_sz) {
                        vnet_hdr = data;
                        data += vnet_hdr_sz;
                        tp_len -= vnet_hdr_sz;
                        if (tp_len < 0 ||
                            __packet_snd_vnet_parse(vnet_hdr, tp_len)) {
                                tp_len = -EINVAL;
                                goto tpacket_error;
                        }
                        copylen = __virtio16_to_cpu(vio_le(),
                                                    vnet_hdr->hdr_len);
                }
                copylen = max_t(int, copylen, dev->hard_header_len);
                skb = sock_alloc_send_skb(&po->sk,
                                hlen + tlen + sizeof(struct sockaddr_ll) +
                                (copylen - dev->hard_header_len),
                                !need_wait, &err);

                if (unlikely(skb == NULL)) {
                        /* we assume the socket was initially writeable ... */
                        if (likely(len_sum > 0))
                                err = len_sum;
                        goto out_status;
                }
                tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto,
                                          addr, hlen, copylen, &sockc);
                if (likely(tp_len >= 0) &&
                    tp_len > dev->mtu + reserve &&
                    !vnet_hdr_sz &&
                    !packet_extra_vlan_len_allowed(dev, skb))
                        tp_len = -EMSGSIZE;

                if (unlikely(tp_len < 0)) {
tpacket_error:
                        if (packet_sock_flag(po, PACKET_SOCK_TP_LOSS)) {
                                __packet_set_status(po, ph,
                                                TP_STATUS_AVAILABLE);
                                packet_increment_head(&po->tx_ring);
                                kfree_skb(skb);
                                continue;
                        } else {
                                status = TP_STATUS_WRONG_FORMAT;
                                err = tp_len;
                                goto out_status;
                        }
                }

                if (vnet_hdr_sz) {
                        if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) {
                                tp_len = -EINVAL;
                                goto tpacket_error;
                        }
                        virtio_net_hdr_set_proto(skb, vnet_hdr);
                }

                skb->destructor = tpacket_destruct_skb;
                __packet_set_status(po, ph, TP_STATUS_SENDING);
                packet_inc_pending(&po->tx_ring);

                status = TP_STATUS_SEND_REQUEST;
                err = packet_xmit(po, skb);
                if (unlikely(err != 0)) {
                        if (err > 0)
                                err = net_xmit_errno(err);
                        if (err && __packet_get_status(po, ph) ==
                                   TP_STATUS_AVAILABLE) {
                                /* skb was destructed already */
                                skb = NULL;
                                goto out_status;
                        }
                        /*
                         * skb was dropped but not destructed yet;
                         * let's treat it like congestion or err < 0
                         */
                        err = 0;
                }
                packet_increment_head(&po->tx_ring);
                len_sum += tp_len;
        } while (likely((ph != NULL) ||
                /* Note: packet_read_pending() might be slow if we have
                 * to call it as it's per_cpu variable, but in fast-path
                 * we already short-circuit the loop with the first
                 * condition, and luckily don't have to go that path
                 * anyway.
                 */
                 (need_wait && packet_read_pending(&po->tx_ring))));

        err = len_sum;
        goto out_put;

out_status:
        __packet_set_status(po, ph, status);
        kfree_skb(skb);
out_put:
        dev_put(dev);
out:
        mutex_unlock(&po->pg_vec_lock);
        return err;
}

static struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
                                        size_t reserve, size_t len,
                                        size_t linear, int noblock,
                                        int *err)
{
        struct sk_buff *skb;

        /* Under a page?  Don't bother with paged skb. */
        if (prepad + len < PAGE_SIZE || !linear)
                linear = len;

        if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
                linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
        skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
                                   err, PAGE_ALLOC_COSTLY_ORDER);
        if (!skb)
                return NULL;

        skb_reserve(skb, reserve);
        skb_put(skb, linear);
        skb->data_len = len - linear;
        skb->len += len - linear;

        return skb;
}

static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
{
        struct sock *sk = sock->sk;
        DECLARE_SOCKADDR(struct sockaddr_ll *, saddr, msg->msg_name);
        struct sk_buff *skb;
        struct net_device *dev;
        __be16 proto;
        unsigned char *addr = NULL;
        int err, reserve = 0;
        struct sockcm_cookie sockc;
        struct virtio_net_hdr vnet_hdr = { 0 };
        int offset = 0;
        struct packet_sock *po = pkt_sk(sk);
        int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz);
        int hlen, tlen, linear;
        int extra_len = 0;

        /*
         *        Get and verify the address.
         */

        if (likely(saddr == NULL)) {
                dev        = packet_cached_dev_get(po);
                proto        = READ_ONCE(po->num);
        } else {
                err = -EINVAL;
                if (msg->msg_namelen < sizeof(struct sockaddr_ll))
                        goto out;
                if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
                        goto out;
                proto        = saddr->sll_protocol;
                dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex);
                if (sock->type == SOCK_DGRAM) {
                        if (dev && msg->msg_namelen < dev->addr_len +
                                   offsetof(struct sockaddr_ll, sll_addr))
                                goto out_unlock;
                        addr = saddr->sll_addr;
                }
        }

        err = -ENXIO;
        if (unlikely(dev == NULL))
                goto out_unlock;
        err = -ENETDOWN;
        if (unlikely(!(dev->flags & IFF_UP)))
                goto out_unlock;

        sockcm_init(&sockc, sk);
        if (msg->msg_controllen) {
                err = sock_cmsg_send(sk, msg, &sockc);
                if (unlikely(err))
                        goto out_unlock;
        }

        if (sock->type == SOCK_RAW)
                reserve = dev->hard_header_len;
        if (vnet_hdr_sz) {
                err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz);
                if (err)
                        goto out_unlock;
        }

        if (unlikely(sock_flag(sk, SOCK_NOFCS))) {
                if (!netif_supports_nofcs(dev)) {
                        err = -EPROTONOSUPPORT;
                        goto out_unlock;
                }
                extra_len = 4; /* We're doing our own CRC */
        }

        err = -EMSGSIZE;
        if (!vnet_hdr.gso_type &&
            (len > dev->mtu + reserve + VLAN_HLEN + extra_len))
                goto out_unlock;

        err = -ENOBUFS;
        hlen = LL_RESERVED_SPACE(dev);
        tlen = dev->needed_tailroom;
        linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len);
        linear = max(linear, min_t(int, len, dev->hard_header_len));
        skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear,
                               msg->msg_flags & MSG_DONTWAIT, &err);
        if (skb == NULL)
                goto out_unlock;

        skb_reset_network_header(skb);

        err = -EINVAL;
        if (sock->type == SOCK_DGRAM) {
                offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len);
                if (unlikely(offset < 0))
                        goto out_free;
        } else if (reserve) {
                skb_reserve(skb, -reserve);
                if (len < reserve + sizeof(struct ipv6hdr) &&
                    dev->min_header_len != dev->hard_header_len)
                        skb_reset_network_header(skb);
        }

        /* Returns -EFAULT on error */
        err = skb_copy_datagram_from_iter(skb, offset, &msg->msg_iter, len);
        if (err)
                goto out_free;

        if ((sock->type == SOCK_RAW &&
             !dev_validate_header(dev, skb->data, len)) || !skb->len) {
                err = -EINVAL;
                goto out_free;
        }

        skb_setup_tx_timestamp(skb, &sockc);

        if (!vnet_hdr.gso_type && (len > dev->mtu + reserve + extra_len) &&
            !packet_extra_vlan_len_allowed(dev, skb)) {
                err = -EMSGSIZE;
                goto out_free;
        }

        skb->protocol = proto;
        skb->dev = dev;
        skb->priority = sockc.priority;
        skb->mark = sockc.mark;
        skb_set_delivery_type_by_clockid(skb, sockc.transmit_time, sk->sk_clockid);

        if (unlikely(extra_len == 4))
                skb->no_fcs = 1;

        packet_parse_headers(skb, sock);

        if (vnet_hdr_sz) {
                err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le());
                if (err)
                        goto out_free;
                len += vnet_hdr_sz;
                virtio_net_hdr_set_proto(skb, &vnet_hdr);
        }

        err = packet_xmit(po, skb);

        if (unlikely(err != 0)) {
                if (err > 0)
                        err = net_xmit_errno(err);
                if (err)
                        goto out_unlock;
        }

        dev_put(dev);

        return len;

out_free:
        kfree_skb(skb);
out_unlock:
        dev_put(dev);
out:
        return err;
}

static int packet_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);

        /* Reading tx_ring.pg_vec without holding pg_vec_lock is racy.
         * tpacket_snd() will redo the check safely.
         */
        if (data_race(po->tx_ring.pg_vec))
                return tpacket_snd(po, msg);

        return packet_snd(sock, msg, len);
}

/*
 *        Close a PACKET socket. This is fairly simple. We immediately go
 *        to 'closed' state and remove our protocol entry in the device list.
 */

static int packet_release(struct socket *sock)
{
        struct sock *sk = sock->sk;
        struct packet_sock *po;
        struct packet_fanout *f;
        struct net *net;
        union tpacket_req_u req_u;

        if (!sk)
                return 0;

        net = sock_net(sk);
        po = pkt_sk(sk);

        mutex_lock(&net->packet.sklist_lock);
        sk_del_node_init_rcu(sk);
        mutex_unlock(&net->packet.sklist_lock);

        sock_prot_inuse_add(net, sk->sk_prot, -1);

        spin_lock(&po->bind_lock);
        unregister_prot_hook(sk, false);
        packet_cached_dev_reset(po);

        if (po->prot_hook.dev) {
                netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);
                po->prot_hook.dev = NULL;
        }
        spin_unlock(&po->bind_lock);

        packet_flush_mclist(sk);

        lock_sock(sk);
        if (po->rx_ring.pg_vec) {
                memset(&req_u, 0, sizeof(req_u));
                packet_set_ring(sk, &req_u, 1, 0);
        }

        if (po->tx_ring.pg_vec) {
                memset(&req_u, 0, sizeof(req_u));
                packet_set_ring(sk, &req_u, 1, 1);
        }
        release_sock(sk);

        f = fanout_release(sk);

        synchronize_net();

        kfree(po->rollover);
        if (f) {
                fanout_release_data(f);
                kvfree(f);
        }
        /*
         *        Now the socket is dead. No more input will appear.
         */
        sock_orphan(sk);
        sock->sk = NULL;

        /* Purge queues */

        skb_queue_purge(&sk->sk_receive_queue);
        packet_free_pending(po);

        sock_put(sk);
        return 0;
}

/*
 *        Attach a packet hook.
 */

static int packet_do_bind(struct sock *sk, const char *name, int ifindex,
                          __be16 proto)
{
        struct packet_sock *po = pkt_sk(sk);
        struct net_device *dev = NULL;
        bool unlisted = false;
        bool need_rehook;
        int ret = 0;

        lock_sock(sk);
        spin_lock(&po->bind_lock);
        if (!proto)
                proto = po->num;

        rcu_read_lock();

        if (po->fanout) {
                ret = -EINVAL;
                goto out_unlock;
        }

        if (name) {
                dev = dev_get_by_name_rcu(sock_net(sk), name);
                if (!dev) {
                        ret = -ENODEV;
                        goto out_unlock;
                }
        } else if (ifindex) {
                dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
                if (!dev) {
                        ret = -ENODEV;
                        goto out_unlock;
                }
        }

        need_rehook = po->prot_hook.type != proto || po->prot_hook.dev != dev;

        if (need_rehook) {
                dev_hold(dev);
                if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
                        rcu_read_unlock();
                        /* prevents packet_notifier() from calling
                         * register_prot_hook()
                         */
                        WRITE_ONCE(po->num, 0);
                        __unregister_prot_hook(sk, true);
                        rcu_read_lock();
                        if (dev)
                                unlisted = !dev_get_by_index_rcu(sock_net(sk),
                                                                 dev->ifindex);
                }

                BUG_ON(packet_sock_flag(po, PACKET_SOCK_RUNNING));
                WRITE_ONCE(po->num, proto);
                po->prot_hook.type = proto;

                netdev_put(po->prot_hook.dev, &po->prot_hook.dev_tracker);

                if (unlikely(unlisted)) {
                        po->prot_hook.dev = NULL;
                        WRITE_ONCE(po->ifindex, -1);
                        packet_cached_dev_reset(po);
                } else {
                        netdev_hold(dev, &po->prot_hook.dev_tracker,
                                    GFP_ATOMIC);
                        po->prot_hook.dev = dev;
                        WRITE_ONCE(po->ifindex, dev ? dev->ifindex : 0);
                        packet_cached_dev_assign(po, dev);
                }
                dev_put(dev);
        }

        if (proto == 0 || !need_rehook)
                goto out_unlock;

        if (!unlisted && (!dev || (dev->flags & IFF_UP))) {
                register_prot_hook(sk);
        } else {
                sk->sk_err = ENETDOWN;
                if (!sock_flag(sk, SOCK_DEAD))
                        sk_error_report(sk);
        }

out_unlock:
        rcu_read_unlock();
        spin_unlock(&po->bind_lock);
        release_sock(sk);
        return ret;
}

/*
 *        Bind a packet socket to a device
 */

static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
                            int addr_len)
{
        struct sock *sk = sock->sk;
        char name[sizeof(uaddr->sa_data_min) + 1];

        /*
         *        Check legality
         */

        if (addr_len != sizeof(struct sockaddr))
                return -EINVAL;
        /* uaddr->sa_data comes from the userspace, it's not guaranteed to be
         * zero-terminated.
         */
        memcpy(name, uaddr->sa_data, sizeof(uaddr->sa_data_min));
        name[sizeof(uaddr->sa_data_min)] = 0;

        return packet_do_bind(sk, name, 0, 0);
}

static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
        struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
        struct sock *sk = sock->sk;

        /*
         *        Check legality
         */

        if (addr_len < sizeof(struct sockaddr_ll))
                return -EINVAL;
        if (sll->sll_family != AF_PACKET)
                return -EINVAL;

        return packet_do_bind(sk, NULL, sll->sll_ifindex, sll->sll_protocol);
}

static struct proto packet_proto = {
        .name          = "PACKET",
        .owner          = THIS_MODULE,
        .obj_size = sizeof(struct packet_sock),
};

/*
 *        Create a packet of type SOCK_PACKET.
 */

static int packet_create(struct net *net, struct socket *sock, int protocol,
                         int kern)
{
        struct sock *sk;
        struct packet_sock *po;
        __be16 proto = (__force __be16)protocol; /* weird, but documented */
        int err;

        if (!ns_capable(net->user_ns, CAP_NET_RAW))
                return -EPERM;
        if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
            sock->type != SOCK_PACKET)
                return -ESOCKTNOSUPPORT;

        sock->state = SS_UNCONNECTED;

        err = -ENOBUFS;
        sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto, kern);
        if (sk == NULL)
                goto out;

        sock->ops = &packet_ops;
        if (sock->type == SOCK_PACKET)
                sock->ops = &packet_ops_spkt;

        po = pkt_sk(sk);
        err = packet_alloc_pending(po);
        if (err)
                goto out_sk_free;

        sock_init_data(sock, sk);

        init_completion(&po->skb_completion);
        sk->sk_family = PF_PACKET;
        po->num = proto;

        packet_cached_dev_reset(po);

        sk->sk_destruct = packet_sock_destruct;

        /*
         *        Attach a protocol block
         */

        spin_lock_init(&po->bind_lock);
        mutex_init(&po->pg_vec_lock);
        po->rollover = NULL;
        po->prot_hook.func = packet_rcv;

        if (sock->type == SOCK_PACKET)
                po->prot_hook.func = packet_rcv_spkt;

        po->prot_hook.af_packet_priv = sk;
        po->prot_hook.af_packet_net = sock_net(sk);

        if (proto) {
                po->prot_hook.type = proto;
                __register_prot_hook(sk);
        }

        mutex_lock(&net->packet.sklist_lock);
        sk_add_node_tail_rcu(sk, &net->packet.sklist);
        mutex_unlock(&net->packet.sklist_lock);

        sock_prot_inuse_add(net, &packet_proto, 1);

        return 0;
out_sk_free:
        sk_free(sk);
out:
        return err;
}

/*
 *        Pull a packet from our receive queue and hand it to the user.
 *        If necessary we block.
 */

static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
                          int flags)
{
        struct sock *sk = sock->sk;
        struct sk_buff *skb;
        int copied, err;
        int vnet_hdr_len = READ_ONCE(pkt_sk(sk)->vnet_hdr_sz);
        unsigned int origlen = 0;

        err = -EINVAL;
        if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
                goto out;

#if 0
        /* What error should we return now? EUNATTACH? */
        if (pkt_sk(sk)->ifindex < 0)
                return -ENODEV;
#endif

        if (flags & MSG_ERRQUEUE) {
                err = sock_recv_errqueue(sk, msg, len,
                                         SOL_PACKET, PACKET_TX_TIMESTAMP);
                goto out;
        }

        /*
         *        Call the generic datagram receiver. This handles all sorts
         *        of horrible races and re-entrancy so we can forget about it
         *        in the protocol layers.
         *
         *        Now it will return ENETDOWN, if device have just gone down,
         *        but then it will block.
         */

        skb = skb_recv_datagram(sk, flags, &err);

        /*
         *        An error occurred so return it. Because skb_recv_datagram()
         *        handles the blocking we don't see and worry about blocking
         *        retries.
         */

        if (skb == NULL)
                goto out;

        packet_rcv_try_clear_pressure(pkt_sk(sk));

        if (vnet_hdr_len) {
                err = packet_rcv_vnet(msg, skb, &len, vnet_hdr_len);
                if (err)
                        goto out_free;
        }

        /* You lose any data beyond the buffer you gave. If it worries
         * a user program they can ask the device for its MTU
         * anyway.
         */
        copied = skb->len;
        if (copied > len) {
                copied = len;
                msg->msg_flags |= MSG_TRUNC;
        }

        err = skb_copy_datagram_msg(skb, 0, msg, copied);
        if (err)
                goto out_free;

        if (sock->type != SOCK_PACKET) {
                struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;

                /* Original length was stored in sockaddr_ll fields */
                origlen = PACKET_SKB_CB(skb)->sa.origlen;
                sll->sll_family = AF_PACKET;
                sll->sll_protocol = (sock->type == SOCK_DGRAM) ?
                        vlan_get_protocol_dgram(skb) : skb->protocol;
        }

        sock_recv_cmsgs(msg, sk, skb);

        if (msg->msg_name) {
                const size_t max_len = min(sizeof(skb->cb),
                                           sizeof(struct sockaddr_storage));
                int copy_len;

                /* If the address length field is there to be filled
                 * in, we fill it in now.
                 */
                if (sock->type == SOCK_PACKET) {
                        __sockaddr_check_size(sizeof(struct sockaddr_pkt));
                        msg->msg_namelen = sizeof(struct sockaddr_pkt);
                        copy_len = msg->msg_namelen;
                } else {
                        struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;

                        msg->msg_namelen = sll->sll_halen +
                                offsetof(struct sockaddr_ll, sll_addr);
                        copy_len = msg->msg_namelen;
                        if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
                                memset(msg->msg_name +
                                       offsetof(struct sockaddr_ll, sll_addr),
                                       0, sizeof(sll->sll_addr));
                                msg->msg_namelen = sizeof(struct sockaddr_ll);
                        }
                }
                if (WARN_ON_ONCE(copy_len > max_len)) {
                        copy_len = max_len;
                        msg->msg_namelen = copy_len;
                }
                memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
        }

        if (packet_sock_flag(pkt_sk(sk), PACKET_SOCK_AUXDATA)) {
                struct tpacket_auxdata aux;

                aux.tp_status = TP_STATUS_USER;
                if (skb->ip_summed == CHECKSUM_PARTIAL)
                        aux.tp_status |= TP_STATUS_CSUMNOTREADY;
                else if (skb->pkt_type != PACKET_OUTGOING &&
                         skb_csum_unnecessary(skb))
                        aux.tp_status |= TP_STATUS_CSUM_VALID;
                if (skb_is_gso(skb) && skb_is_gso_tcp(skb))
                        aux.tp_status |= TP_STATUS_GSO_TCP;

                aux.tp_len = origlen;
                aux.tp_snaplen = skb->len;
                aux.tp_mac = 0;
                aux.tp_net = skb_network_offset(skb);
                if (skb_vlan_tag_present(skb)) {
                        aux.tp_vlan_tci = skb_vlan_tag_get(skb);
                        aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
                        aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
                } else if (unlikely(sock->type == SOCK_DGRAM && eth_type_vlan(skb->protocol))) {
                        struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;
                        struct net_device *dev;

                        rcu_read_lock();
                        dev = dev_get_by_index_rcu(sock_net(sk), sll->sll_ifindex);
                        if (dev) {
                                aux.tp_vlan_tci = vlan_get_tci(skb, dev);
                                aux.tp_vlan_tpid = ntohs(skb->protocol);
                                aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
                        } else {
                                aux.tp_vlan_tci = 0;
                                aux.tp_vlan_tpid = 0;
                        }
                        rcu_read_unlock();
                } else {
                        aux.tp_vlan_tci = 0;
                        aux.tp_vlan_tpid = 0;
                }
                put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
        }

        /*
         *        Free or return the buffer as appropriate. Again this
         *        hides all the races and re-entrancy issues from us.
         */
        err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);

out_free:
        skb_free_datagram(sk, skb);
out:
        return err;
}

static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
                               int peer)
{
        struct net_device *dev;
        struct sock *sk        = sock->sk;

        if (peer)
                return -EOPNOTSUPP;

        uaddr->sa_family = AF_PACKET;
        memset(uaddr->sa_data, 0, sizeof(uaddr->sa_data_min));
        rcu_read_lock();
        dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
        if (dev)
                strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data_min));
        rcu_read_unlock();

        return sizeof(*uaddr);
}

static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
                          int peer)
{
        struct net_device *dev;
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
        int ifindex;

        if (peer)
                return -EOPNOTSUPP;

        ifindex = READ_ONCE(po->ifindex);
        sll->sll_family = AF_PACKET;
        sll->sll_ifindex = ifindex;
        sll->sll_protocol = READ_ONCE(po->num);
        sll->sll_pkttype = 0;
        rcu_read_lock();
        dev = dev_get_by_index_rcu(sock_net(sk), ifindex);
        if (dev) {
                sll->sll_hatype = dev->type;
                sll->sll_halen = dev->addr_len;

                /* Let __fortify_memcpy_chk() know the actual buffer size. */
                memcpy(((struct sockaddr_storage *)sll)->__data +
                       offsetof(struct sockaddr_ll, sll_addr) -
                       offsetofend(struct sockaddr_ll, sll_family),
                       dev->dev_addr, dev->addr_len);
        } else {
                sll->sll_hatype = 0;        /* Bad: we have no ARPHRD_UNSPEC */
                sll->sll_halen = 0;
        }
        rcu_read_unlock();

        return offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
}

static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
                         int what)
{
        switch (i->type) {
        case PACKET_MR_MULTICAST:
                if (i->alen != dev->addr_len)
                        return -EINVAL;
                if (what > 0)
                        return dev_mc_add(dev, i->addr);
                else
                        return dev_mc_del(dev, i->addr);
                break;
        case PACKET_MR_PROMISC:
                return dev_set_promiscuity(dev, what);
        case PACKET_MR_ALLMULTI:
                return dev_set_allmulti(dev, what);
        case PACKET_MR_UNICAST:
                if (i->alen != dev->addr_len)
                        return -EINVAL;
                if (what > 0)
                        return dev_uc_add(dev, i->addr);
                else
                        return dev_uc_del(dev, i->addr);
                break;
        default:
                break;
        }
        return 0;
}

static void packet_dev_mclist_delete(struct net_device *dev,
                                     struct packet_mclist **mlp,
                                     struct list_head *list)
{
        struct packet_mclist *ml;

        while ((ml = *mlp) != NULL) {
                if (ml->ifindex == dev->ifindex) {
                        list_add(&ml->remove_list, list);
                        *mlp = ml->next;
                } else
                        mlp = &ml->next;
        }
}

static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
{
        struct packet_sock *po = pkt_sk(sk);
        struct packet_mclist *ml, *i;
        struct net_device *dev;
        int err;

        rtnl_lock();

        err = -ENODEV;
        dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
        if (!dev)
                goto done;

        err = -EINVAL;
        if (mreq->mr_alen > dev->addr_len)
                goto done;

        err = -ENOBUFS;
        i = kmalloc(sizeof(*i), GFP_KERNEL);
        if (i == NULL)
                goto done;

        err = 0;
        for (ml = po->mclist; ml; ml = ml->next) {
                if (ml->ifindex == mreq->mr_ifindex &&
                    ml->type == mreq->mr_type &&
                    ml->alen == mreq->mr_alen &&
                    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
                        ml->count++;
                        /* Free the new element ... */
                        kfree(i);
                        goto done;
                }
        }

        i->type = mreq->mr_type;
        i->ifindex = mreq->mr_ifindex;
        i->alen = mreq->mr_alen;
        memcpy(i->addr, mreq->mr_address, i->alen);
        memset(i->addr + i->alen, 0, sizeof(i->addr) - i->alen);
        i->count = 1;
        INIT_LIST_HEAD(&i->remove_list);
        i->next = po->mclist;
        po->mclist = i;
        err = packet_dev_mc(dev, i, 1);
        if (err) {
                po->mclist = i->next;
                kfree(i);
        }

done:
        rtnl_unlock();
        return err;
}

static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
{
        struct packet_mclist *ml, **mlp;

        rtnl_lock();

        for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
                if (ml->ifindex == mreq->mr_ifindex &&
                    ml->type == mreq->mr_type &&
                    ml->alen == mreq->mr_alen &&
                    memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
                        if (--ml->count == 0) {
                                struct net_device *dev;
                                *mlp = ml->next;
                                dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
                                if (dev)
                                        packet_dev_mc(dev, ml, -1);
                                kfree(ml);
                        }
                        break;
                }
        }
        rtnl_unlock();
        return 0;
}

static void packet_flush_mclist(struct sock *sk)
{
        struct packet_sock *po = pkt_sk(sk);
        struct packet_mclist *ml;

        if (!po->mclist)
                return;

        rtnl_lock();
        while ((ml = po->mclist) != NULL) {
                struct net_device *dev;

                po->mclist = ml->next;
                dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
                if (dev != NULL)
                        packet_dev_mc(dev, ml, -1);
                kfree(ml);
        }
        rtnl_unlock();
}

static int
packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
                  unsigned int optlen)
{
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        int ret;

        if (level != SOL_PACKET)
                return -ENOPROTOOPT;

        switch (optname) {
        case PACKET_ADD_MEMBERSHIP:
        case PACKET_DROP_MEMBERSHIP:
        {
                struct packet_mreq_max mreq;
                int len = optlen;
                memset(&mreq, 0, sizeof(mreq));
                if (len < sizeof(struct packet_mreq))
                        return -EINVAL;
                if (len > sizeof(mreq))
                        len = sizeof(mreq);
                if (copy_from_sockptr(&mreq, optval, len))
                        return -EFAULT;
                if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
                        return -EINVAL;
                if (optname == PACKET_ADD_MEMBERSHIP)
                        ret = packet_mc_add(sk, &mreq);
                else
                        ret = packet_mc_drop(sk, &mreq);
                return ret;
        }

        case PACKET_RX_RING:
        case PACKET_TX_RING:
        {
                union tpacket_req_u req_u;

                ret = -EINVAL;
                lock_sock(sk);
                switch (po->tp_version) {
                case TPACKET_V1:
                case TPACKET_V2:
                        if (optlen < sizeof(req_u.req))
                                break;
                        ret = copy_from_sockptr(&req_u.req, optval,
                                                sizeof(req_u.req)) ?
                                                -EINVAL : 0;
                        break;
                case TPACKET_V3:
                default:
                        if (optlen < sizeof(req_u.req3))
                                break;
                        ret = copy_from_sockptr(&req_u.req3, optval,
                                                sizeof(req_u.req3)) ?
                                                -EINVAL : 0;
                        break;
                }
                if (!ret)
                        ret = packet_set_ring(sk, &req_u, 0,
                                              optname == PACKET_TX_RING);
                release_sock(sk);
                return ret;
        }
        case PACKET_COPY_THRESH:
        {
                int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                WRITE_ONCE(pkt_sk(sk)->copy_thresh, val);
                return 0;
        }
        case PACKET_VERSION:
        {
                int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;
                switch (val) {
                case TPACKET_V1:
                case TPACKET_V2:
                case TPACKET_V3:
                        break;
                default:
                        return -EINVAL;
                }
                lock_sock(sk);
                if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
                        ret = -EBUSY;
                } else {
                        po->tp_version = val;
                        ret = 0;
                }
                release_sock(sk);
                return ret;
        }
        case PACKET_RESERVE:
        {
                unsigned int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;
                if (val > INT_MAX)
                        return -EINVAL;
                lock_sock(sk);
                if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
                        ret = -EBUSY;
                } else {
                        po->tp_reserve = val;
                        ret = 0;
                }
                release_sock(sk);
                return ret;
        }
        case PACKET_LOSS:
        {
                unsigned int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                lock_sock(sk);
                if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
                        ret = -EBUSY;
                } else {
                        packet_sock_flag_set(po, PACKET_SOCK_TP_LOSS, val);
                        ret = 0;
                }
                release_sock(sk);
                return ret;
        }
        case PACKET_AUXDATA:
        {
                int val;

                if (optlen < sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                packet_sock_flag_set(po, PACKET_SOCK_AUXDATA, val);
                return 0;
        }
        case PACKET_ORIGDEV:
        {
                int val;

                if (optlen < sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                packet_sock_flag_set(po, PACKET_SOCK_ORIGDEV, val);
                return 0;
        }
        case PACKET_VNET_HDR:
        case PACKET_VNET_HDR_SZ:
        {
                int val, hdr_len;

                if (sock->type != SOCK_RAW)
                        return -EINVAL;
                if (optlen < sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                if (optname == PACKET_VNET_HDR_SZ) {
                        if (val && val != sizeof(struct virtio_net_hdr) &&
                            val != sizeof(struct virtio_net_hdr_mrg_rxbuf))
                                return -EINVAL;
                        hdr_len = val;
                } else {
                        hdr_len = val ? sizeof(struct virtio_net_hdr) : 0;
                }
                lock_sock(sk);
                if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) {
                        ret = -EBUSY;
                } else {
                        WRITE_ONCE(po->vnet_hdr_sz, hdr_len);
                        ret = 0;
                }
                release_sock(sk);
                return ret;
        }
        case PACKET_TIMESTAMP:
        {
                int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                WRITE_ONCE(po->tp_tstamp, val);
                return 0;
        }
        case PACKET_FANOUT:
        {
                struct fanout_args args = { 0 };

                if (optlen != sizeof(int) && optlen != sizeof(args))
                        return -EINVAL;
                if (copy_from_sockptr(&args, optval, optlen))
                        return -EFAULT;

                return fanout_add(sk, &args);
        }
        case PACKET_FANOUT_DATA:
        {
                /* Paired with the WRITE_ONCE() in fanout_add() */
                if (!READ_ONCE(po->fanout))
                        return -EINVAL;

                return fanout_set_data(po, optval, optlen);
        }
        case PACKET_IGNORE_OUTGOING:
        {
                int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;
                if (val < 0 || val > 1)
                        return -EINVAL;

                WRITE_ONCE(po->prot_hook.ignore_outgoing, !!val);
                return 0;
        }
        case PACKET_TX_HAS_OFF:
        {
                unsigned int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                lock_sock(sk);
                if (!po->rx_ring.pg_vec && !po->tx_ring.pg_vec)
                        packet_sock_flag_set(po, PACKET_SOCK_TX_HAS_OFF, val);

                release_sock(sk);
                return 0;
        }
        case PACKET_QDISC_BYPASS:
        {
                int val;

                if (optlen != sizeof(val))
                        return -EINVAL;
                if (copy_from_sockptr(&val, optval, sizeof(val)))
                        return -EFAULT;

                packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val);
                return 0;
        }
        default:
                return -ENOPROTOOPT;
        }
}

static int packet_getsockopt(struct socket *sock, int level, int optname,
                             char __user *optval, int __user *optlen)
{
        int len;
        int val, lv = sizeof(val);
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        void *data = &val;
        union tpacket_stats_u st;
        struct tpacket_rollover_stats rstats;
        int drops;

        if (level != SOL_PACKET)
                return -ENOPROTOOPT;

        if (get_user(len, optlen))
                return -EFAULT;

        if (len < 0)
                return -EINVAL;

        switch (optname) {
        case PACKET_STATISTICS:
                spin_lock_bh(&sk->sk_receive_queue.lock);
                memcpy(&st, &po->stats, sizeof(st));
                memset(&po->stats, 0, sizeof(po->stats));
                spin_unlock_bh(&sk->sk_receive_queue.lock);
                drops = atomic_xchg(&po->tp_drops, 0);

                if (po->tp_version == TPACKET_V3) {
                        lv = sizeof(struct tpacket_stats_v3);
                        st.stats3.tp_drops = drops;
                        st.stats3.tp_packets += drops;
                        data = &st.stats3;
                } else {
                        lv = sizeof(struct tpacket_stats);
                        st.stats1.tp_drops = drops;
                        st.stats1.tp_packets += drops;
                        data = &st.stats1;
                }

                break;
        case PACKET_AUXDATA:
                val = packet_sock_flag(po, PACKET_SOCK_AUXDATA);
                break;
        case PACKET_ORIGDEV:
                val = packet_sock_flag(po, PACKET_SOCK_ORIGDEV);
                break;
        case PACKET_VNET_HDR:
                val = !!READ_ONCE(po->vnet_hdr_sz);
                break;
        case PACKET_VNET_HDR_SZ:
                val = READ_ONCE(po->vnet_hdr_sz);
                break;
        case PACKET_COPY_THRESH:
                val = READ_ONCE(pkt_sk(sk)->copy_thresh);
                break;
        case PACKET_VERSION:
                val = po->tp_version;
                break;
        case PACKET_HDRLEN:
                if (len > sizeof(int))
                        len = sizeof(int);
                if (len < sizeof(int))
                        return -EINVAL;
                if (copy_from_user(&val, optval, len))
                        return -EFAULT;
                switch (val) {
                case TPACKET_V1:
                        val = sizeof(struct tpacket_hdr);
                        break;
                case TPACKET_V2:
                        val = sizeof(struct tpacket2_hdr);
                        break;
                case TPACKET_V3:
                        val = sizeof(struct tpacket3_hdr);
                        break;
                default:
                        return -EINVAL;
                }
                break;
        case PACKET_RESERVE:
                val = po->tp_reserve;
                break;
        case PACKET_LOSS:
                val = packet_sock_flag(po, PACKET_SOCK_TP_LOSS);
                break;
        case PACKET_TIMESTAMP:
                val = READ_ONCE(po->tp_tstamp);
                break;
        case PACKET_FANOUT:
                val = (po->fanout ?
                       ((u32)po->fanout->id |
                        ((u32)po->fanout->type << 16) |
                        ((u32)po->fanout->flags << 24)) :
                       0);
                break;
        case PACKET_IGNORE_OUTGOING:
                val = READ_ONCE(po->prot_hook.ignore_outgoing);
                break;
        case PACKET_ROLLOVER_STATS:
                if (!po->rollover)
                        return -EINVAL;
                rstats.tp_all = atomic_long_read(&po->rollover->num);
                rstats.tp_huge = atomic_long_read(&po->rollover->num_huge);
                rstats.tp_failed = atomic_long_read(&po->rollover->num_failed);
                data = &rstats;
                lv = sizeof(rstats);
                break;
        case PACKET_TX_HAS_OFF:
                val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF);
                break;
        case PACKET_QDISC_BYPASS:
                val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS);
                break;
        default:
                return -ENOPROTOOPT;
        }

        if (len > lv)
                len = lv;
        if (put_user(len, optlen))
                return -EFAULT;
        if (copy_to_user(optval, data, len))
                return -EFAULT;
        return 0;
}

static int packet_notifier(struct notifier_block *this,
                           unsigned long msg, void *ptr)
{
        struct net_device *dev = netdev_notifier_info_to_dev(ptr);
        struct net *net = dev_net(dev);
        struct packet_mclist *ml, *tmp;
        LIST_HEAD(mclist);
        struct sock *sk;

        rcu_read_lock();
        sk_for_each_rcu(sk, &net->packet.sklist) {
                struct packet_sock *po = pkt_sk(sk);

                switch (msg) {
                case NETDEV_UNREGISTER:
                        if (po->mclist)
                                packet_dev_mclist_delete(dev, &po->mclist,
                                                         &mclist);
                        fallthrough;

                case NETDEV_DOWN:
                        if (dev->ifindex == po->ifindex) {
                                spin_lock(&po->bind_lock);
                                if (packet_sock_flag(po, PACKET_SOCK_RUNNING)) {
                                        __unregister_prot_hook(sk, false);
                                        sk->sk_err = ENETDOWN;
                                        if (!sock_flag(sk, SOCK_DEAD))
                                                sk_error_report(sk);
                                }
                                if (msg == NETDEV_UNREGISTER) {
                                        packet_cached_dev_reset(po);
                                        WRITE_ONCE(po->ifindex, -1);
                                        netdev_put(po->prot_hook.dev,
                                                   &po->prot_hook.dev_tracker);
                                        po->prot_hook.dev = NULL;
                                }
                                spin_unlock(&po->bind_lock);
                        }
                        break;
                case NETDEV_UP:
                        if (dev->ifindex == po->ifindex) {
                                spin_lock(&po->bind_lock);
                                if (po->num)
                                        register_prot_hook(sk);
                                spin_unlock(&po->bind_lock);
                        }
                        break;
                }
        }
        rcu_read_unlock();

        /* packet_dev_mc might grab instance locks so can't run under rcu */
        list_for_each_entry_safe(ml, tmp, &mclist, remove_list) {
                packet_dev_mc(dev, ml, -1);
                kfree(ml);
        }

        return NOTIFY_DONE;
}


static int packet_ioctl(struct socket *sock, unsigned int cmd,
                        unsigned long arg)
{
        struct sock *sk = sock->sk;

        switch (cmd) {
        case SIOCOUTQ:
        {
                int amount = sk_wmem_alloc_get(sk);

                return put_user(amount, (int __user *)arg);
        }
        case SIOCINQ:
        {
                struct sk_buff *skb;
                int amount = 0;

                spin_lock_bh(&sk->sk_receive_queue.lock);
                skb = skb_peek(&sk->sk_receive_queue);
                if (skb)
                        amount = skb->len;
                spin_unlock_bh(&sk->sk_receive_queue.lock);
                return put_user(amount, (int __user *)arg);
        }
#ifdef CONFIG_INET
        case SIOCADDRT:
        case SIOCDELRT:
        case SIOCDARP:
        case SIOCGARP:
        case SIOCSARP:
        case SIOCGIFADDR:
        case SIOCSIFADDR:
        case SIOCGIFBRDADDR:
        case SIOCSIFBRDADDR:
        case SIOCGIFNETMASK:
        case SIOCSIFNETMASK:
        case SIOCGIFDSTADDR:
        case SIOCSIFDSTADDR:
        case SIOCSIFFLAGS:
                return inet_dgram_ops.ioctl(sock, cmd, arg);
#endif

        default:
                return -ENOIOCTLCMD;
        }
        return 0;
}

static __poll_t packet_poll(struct file *file, struct socket *sock,
                                poll_table *wait)
{
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        __poll_t mask = datagram_poll(file, sock, wait);

        spin_lock_bh(&sk->sk_receive_queue.lock);
        if (po->rx_ring.pg_vec) {
                if (!packet_previous_rx_frame(po, &po->rx_ring,
                        TP_STATUS_KERNEL))
                        mask |= EPOLLIN | EPOLLRDNORM;
        }
        packet_rcv_try_clear_pressure(po);
        spin_unlock_bh(&sk->sk_receive_queue.lock);
        spin_lock_bh(&sk->sk_write_queue.lock);
        if (po->tx_ring.pg_vec) {
                if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
                        mask |= EPOLLOUT | EPOLLWRNORM;
        }
        spin_unlock_bh(&sk->sk_write_queue.lock);
        return mask;
}


/* Dirty? Well, I still did not learn better way to account
 * for user mmaps.
 */

static void packet_mm_open(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct socket *sock = file->private_data;
        struct sock *sk = sock->sk;

        if (sk)
                atomic_long_inc(&pkt_sk(sk)->mapped);
}

static void packet_mm_close(struct vm_area_struct *vma)
{
        struct file *file = vma->vm_file;
        struct socket *sock = file->private_data;
        struct sock *sk = sock->sk;

        if (sk)
                atomic_long_dec(&pkt_sk(sk)->mapped);
}

static const struct vm_operations_struct packet_mmap_ops = {
        .open        =        packet_mm_open,
        .close        =        packet_mm_close,
};

static void free_pg_vec(struct pgv *pg_vec, unsigned int order,
                        unsigned int len)
{
        int i;

        for (i = 0; i < len; i++) {
                if (likely(pg_vec[i].buffer)) {
                        if (is_vmalloc_addr(pg_vec[i].buffer))
                                vfree(pg_vec[i].buffer);
                        else
                                free_pages((unsigned long)pg_vec[i].buffer,
                                           order);
                        pg_vec[i].buffer = NULL;
                }
        }
        kfree(pg_vec);
}

static char *alloc_one_pg_vec_page(unsigned long order)
{
        char *buffer;
        gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP |
                          __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY;

        buffer = (char *) __get_free_pages(gfp_flags, order);
        if (buffer)
                return buffer;

        /* __get_free_pages failed, fall back to vmalloc */
        buffer = vzalloc(array_size((1 << order), PAGE_SIZE));
        if (buffer)
                return buffer;

        /* vmalloc failed, lets dig into swap here */
        gfp_flags &= ~__GFP_NORETRY;
        buffer = (char *) __get_free_pages(gfp_flags, order);
        if (buffer)
                return buffer;

        /* complete and utter failure */
        return NULL;
}

static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order)
{
        unsigned int block_nr = req->tp_block_nr;
        struct pgv *pg_vec;
        int i;

        pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN);
        if (unlikely(!pg_vec))
                goto out;

        for (i = 0; i < block_nr; i++) {
                pg_vec[i].buffer = alloc_one_pg_vec_page(order);
                if (unlikely(!pg_vec[i].buffer))
                        goto out_free_pgvec;
        }

out:
        return pg_vec;

out_free_pgvec:
        free_pg_vec(pg_vec, order, block_nr);
        pg_vec = NULL;
        goto out;
}

static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u,
                int closing, int tx_ring)
{
        struct pgv *pg_vec = NULL;
        struct packet_sock *po = pkt_sk(sk);
        unsigned long *rx_owner_map = NULL;
        int was_running, order = 0;
        struct packet_ring_buffer *rb;
        struct sk_buff_head *rb_queue;
        __be16 num;
        int err;
        /* Added to avoid minimal code churn */
        struct tpacket_req *req = &req_u->req;

        rb = tx_ring ? &po->tx_ring : &po->rx_ring;
        rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;

        err = -EBUSY;
        if (!closing) {
                if (atomic_long_read(&po->mapped))
                        goto out;
                if (packet_read_pending(rb))
                        goto out;
        }

        if (req->tp_block_nr) {
                unsigned int min_frame_size;

                /* Sanity tests and some calculations */
                err = -EBUSY;
                if (unlikely(rb->pg_vec))
                        goto out;

                switch (po->tp_version) {
                case TPACKET_V1:
                        po->tp_hdrlen = TPACKET_HDRLEN;
                        break;
                case TPACKET_V2:
                        po->tp_hdrlen = TPACKET2_HDRLEN;
                        break;
                case TPACKET_V3:
                        po->tp_hdrlen = TPACKET3_HDRLEN;
                        break;
                }

                err = -EINVAL;
                if (unlikely((int)req->tp_block_size <= 0))
                        goto out;
                if (unlikely(!PAGE_ALIGNED(req->tp_block_size)))
                        goto out;
                min_frame_size = po->tp_hdrlen + po->tp_reserve;
                if (po->tp_version >= TPACKET_V3 &&
                    req->tp_block_size <
                    BLK_PLUS_PRIV((u64)req_u->req3.tp_sizeof_priv) + min_frame_size)
                        goto out;
                if (unlikely(req->tp_frame_size < min_frame_size))
                        goto out;
                if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
                        goto out;

                rb->frames_per_block = req->tp_block_size / req->tp_frame_size;
                if (unlikely(rb->frames_per_block == 0))
                        goto out;
                if (unlikely(rb->frames_per_block > UINT_MAX / req->tp_block_nr))
                        goto out;
                if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
                                        req->tp_frame_nr))
                        goto out;

                err = -ENOMEM;
                order = get_order(req->tp_block_size);
                pg_vec = alloc_pg_vec(req, order);
                if (unlikely(!pg_vec))
                        goto out;
                switch (po->tp_version) {
                case TPACKET_V3:
                        /* Block transmit is not supported yet */
                        if (!tx_ring) {
                                init_prb_bdqc(po, rb, pg_vec, req_u);
                        } else {
                                struct tpacket_req3 *req3 = &req_u->req3;

                                if (req3->tp_retire_blk_tov ||
                                    req3->tp_sizeof_priv ||
                                    req3->tp_feature_req_word) {
                                        err = -EINVAL;
                                        goto out_free_pg_vec;
                                }
                        }
                        break;
                default:
                        if (!tx_ring) {
                                rx_owner_map = bitmap_alloc(req->tp_frame_nr,
                                        GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
                                if (!rx_owner_map)
                                        goto out_free_pg_vec;
                        }
                        break;
                }
        }
        /* Done */
        else {
                err = -EINVAL;
                if (unlikely(req->tp_frame_nr))
                        goto out;
        }


        /* Detach socket from network */
        spin_lock(&po->bind_lock);
        was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING);
        num = po->num;
        if (was_running) {
                WRITE_ONCE(po->num, 0);
                __unregister_prot_hook(sk, false);
        }
        spin_unlock(&po->bind_lock);

        synchronize_net();

        err = -EBUSY;
        mutex_lock(&po->pg_vec_lock);
        if (closing || atomic_long_read(&po->mapped) == 0) {
                err = 0;
                spin_lock_bh(&rb_queue->lock);
                swap(rb->pg_vec, pg_vec);
                if (po->tp_version <= TPACKET_V2)
                        swap(rb->rx_owner_map, rx_owner_map);
                rb->frame_max = (req->tp_frame_nr - 1);
                rb->head = 0;
                rb->frame_size = req->tp_frame_size;
                spin_unlock_bh(&rb_queue->lock);

                swap(rb->pg_vec_order, order);
                swap(rb->pg_vec_len, req->tp_block_nr);

                rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
                po->prot_hook.func = (po->rx_ring.pg_vec) ?
                                                tpacket_rcv : packet_rcv;
                skb_queue_purge(rb_queue);
                if (atomic_long_read(&po->mapped))
                        pr_err("packet_mmap: vma is busy: %ld\n",
                               atomic_long_read(&po->mapped));
        }
        mutex_unlock(&po->pg_vec_lock);

        spin_lock(&po->bind_lock);
        if (was_running) {
                WRITE_ONCE(po->num, num);
                register_prot_hook(sk);
        }
        spin_unlock(&po->bind_lock);
        if (pg_vec && (po->tp_version > TPACKET_V2)) {
                /* Because we don't support block-based V3 on tx-ring */
                if (!tx_ring)
                        prb_shutdown_retire_blk_timer(po, rb_queue);
        }

out_free_pg_vec:
        if (pg_vec) {
                bitmap_free(rx_owner_map);
                free_pg_vec(pg_vec, order, req->tp_block_nr);
        }
out:
        return err;
}

static int packet_mmap(struct file *file, struct socket *sock,
                struct vm_area_struct *vma)
{
        struct sock *sk = sock->sk;
        struct packet_sock *po = pkt_sk(sk);
        unsigned long size, expected_size;
        struct packet_ring_buffer *rb;
        unsigned long start;
        int err = -EINVAL;
        int i;

        if (vma->vm_pgoff)
                return -EINVAL;

        mutex_lock(&po->pg_vec_lock);

        expected_size = 0;
        for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
                if (rb->pg_vec) {
                        expected_size += rb->pg_vec_len
                                                * rb->pg_vec_pages
                                                * PAGE_SIZE;
                }
        }

        if (expected_size == 0)
                goto out;

        size = vma->vm_end - vma->vm_start;
        if (size != expected_size)
                goto out;

        start = vma->vm_start;
        for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
                if (rb->pg_vec == NULL)
                        continue;

                for (i = 0; i < rb->pg_vec_len; i++) {
                        struct page *page;
                        void *kaddr = rb->pg_vec[i].buffer;
                        int pg_num;

                        for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) {
                                page = pgv_to_page(kaddr);
                                err = vm_insert_page(vma, start, page);
                                if (unlikely(err))
                                        goto out;
                                start += PAGE_SIZE;
                                kaddr += PAGE_SIZE;
                        }
                }
        }

        atomic_long_inc(&po->mapped);
        vma->vm_ops = &packet_mmap_ops;
        err = 0;

out:
        mutex_unlock(&po->pg_vec_lock);
        return err;
}

static const struct proto_ops packet_ops_spkt = {
        .family =        PF_PACKET,
        .owner =        THIS_MODULE,
        .release =        packet_release,
        .bind =                packet_bind_spkt,
        .connect =        sock_no_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        sock_no_accept,
        .getname =        packet_getname_spkt,
        .poll =                datagram_poll,
        .ioctl =        packet_ioctl,
        .gettstamp =        sock_gettstamp,
        .listen =        sock_no_listen,
        .shutdown =        sock_no_shutdown,
        .sendmsg =        packet_sendmsg_spkt,
        .recvmsg =        packet_recvmsg,
        .mmap =                sock_no_mmap,
};

static const struct proto_ops packet_ops = {
        .family =        PF_PACKET,
        .owner =        THIS_MODULE,
        .release =        packet_release,
        .bind =                packet_bind,
        .connect =        sock_no_connect,
        .socketpair =        sock_no_socketpair,
        .accept =        sock_no_accept,
        .getname =        packet_getname,
        .poll =                packet_poll,
        .ioctl =        packet_ioctl,
        .gettstamp =        sock_gettstamp,
        .listen =        sock_no_listen,
        .shutdown =        sock_no_shutdown,
        .setsockopt =        packet_setsockopt,
        .getsockopt =        packet_getsockopt,
        .sendmsg =        packet_sendmsg,
        .recvmsg =        packet_recvmsg,
        .mmap =                packet_mmap,
};

static const struct net_proto_family packet_family_ops = {
        .family =        PF_PACKET,
        .create =        packet_create,
        .owner        =        THIS_MODULE,
};

static struct notifier_block packet_netdev_notifier = {
        .notifier_call =        packet_notifier,
};

#ifdef CONFIG_PROC_FS

static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(RCU)
{
        struct net *net = seq_file_net(seq);

        rcu_read_lock();
        return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
}

static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct net *net = seq_file_net(seq);
        return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
}

static void packet_seq_stop(struct seq_file *seq, void *v)
        __releases(RCU)
{
        rcu_read_unlock();
}

static int packet_seq_show(struct seq_file *seq, void *v)
{
        if (v == SEQ_START_TOKEN)
                seq_printf(seq,
                           "%*sRefCnt Type Proto  Iface R Rmem   User   Inode\n",
                           IS_ENABLED(CONFIG_64BIT) ? -17 : -9, "sk");
        else {
                struct sock *s = sk_entry(v);
                const struct packet_sock *po = pkt_sk(s);

                seq_printf(seq,
                           "%pK %-6d %-4d %04x   %-5d %1d %-6u %-6u %-6lu\n",
                           s,
                           refcount_read(&s->sk_refcnt),
                           s->sk_type,
                           ntohs(READ_ONCE(po->num)),
                           READ_ONCE(po->ifindex),
                           packet_sock_flag(po, PACKET_SOCK_RUNNING),
                           atomic_read(&s->sk_rmem_alloc),
                           from_kuid_munged(seq_user_ns(seq), sock_i_uid(s)),
                           sock_i_ino(s));
        }

        return 0;
}

static const struct seq_operations packet_seq_ops = {
        .start        = packet_seq_start,
        .next        = packet_seq_next,
        .stop        = packet_seq_stop,
        .show        = packet_seq_show,
};
#endif

static int __net_init packet_net_init(struct net *net)
{
        mutex_init(&net->packet.sklist_lock);
        INIT_HLIST_HEAD(&net->packet.sklist);

#ifdef CONFIG_PROC_FS
        if (!proc_create_net("packet", 0, net->proc_net, &packet_seq_ops,
                        sizeof(struct seq_net_private)))
                return -ENOMEM;
#endif /* CONFIG_PROC_FS */

        return 0;
}

static void __net_exit packet_net_exit(struct net *net)
{
        remove_proc_entry("packet", net->proc_net);
        WARN_ON_ONCE(!hlist_empty(&net->packet.sklist));
}

static struct pernet_operations packet_net_ops = {
        .init = packet_net_init,
        .exit = packet_net_exit,
};


static void __exit packet_exit(void)
{
        sock_unregister(PF_PACKET);
        proto_unregister(&packet_proto);
        unregister_netdevice_notifier(&packet_netdev_notifier);
        unregister_pernet_subsys(&packet_net_ops);
}

static int __init packet_init(void)
{
        int rc;

        rc = register_pernet_subsys(&packet_net_ops);
        if (rc)
                goto out;
        rc = register_netdevice_notifier(&packet_netdev_notifier);
        if (rc)
                goto out_pernet;
        rc = proto_register(&packet_proto, 0);
        if (rc)
                goto out_notifier;
        rc = sock_register(&packet_family_ops);
        if (rc)
                goto out_proto;

        return 0;

out_proto:
        proto_unregister(&packet_proto);
out_notifier:
        unregister_netdevice_notifier(&packet_netdev_notifier);
out_pernet:
        unregister_pernet_subsys(&packet_net_ops);
out:
        return rc;
}

module_init(packet_init);
module_exit(packet_exit);
MODULE_DESCRIPTION("Packet socket support (AF_PACKET)");
MODULE_LICENSE("GPL");
MODULE_ALIAS_NETPROTO(PF_PACKET);

































































































































































































































































































































































































































































































 1681 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_PREEMPT_H
#define __LINUX_PREEMPT_H

/*
 * include/linux/preempt.h - macros for accessing and manipulating
 * preempt_count (used for kernel preemption, interrupt count, etc.)
 */

#include <linux/linkage.h>
#include <linux/cleanup.h>
#include <linux/types.h>

/*
 * We put the hardirq and softirq counter into the preemption
 * counter. The bitmask has the following meaning:
 *
 * - bits 0-7 are the preemption count (max preemption depth: 256)
 * - bits 8-15 are the softirq count (max # of softirqs: 256)
 *
 * The hardirq count could in theory be the same as the number of
 * interrupts in the system, but we run all interrupt handlers with
 * interrupts disabled, so we cannot have nesting interrupts. Though
 * there are a few palaeontologic drivers which reenable interrupts in
 * the handler, so we need more than one bit here.
 *
 *         PREEMPT_MASK:        0x000000ff
 *         SOFTIRQ_MASK:        0x0000ff00
 *         HARDIRQ_MASK:        0x000f0000
 *             NMI_MASK:        0x00f00000
 * PREEMPT_NEED_RESCHED:        0x80000000
 */
#define PREEMPT_BITS        8
#define SOFTIRQ_BITS        8
#define HARDIRQ_BITS        4
#define NMI_BITS        4

#define PREEMPT_SHIFT        0
#define SOFTIRQ_SHIFT        (PREEMPT_SHIFT + PREEMPT_BITS)
#define HARDIRQ_SHIFT        (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
#define NMI_SHIFT        (HARDIRQ_SHIFT + HARDIRQ_BITS)

#define __IRQ_MASK(x)        ((1UL << (x))-1)

#define PREEMPT_MASK        (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
#define SOFTIRQ_MASK        (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
#define HARDIRQ_MASK        (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK        (__IRQ_MASK(NMI_BITS)     << NMI_SHIFT)

#define PREEMPT_OFFSET        (1UL << PREEMPT_SHIFT)
#define SOFTIRQ_OFFSET        (1UL << SOFTIRQ_SHIFT)
#define HARDIRQ_OFFSET        (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET        (1UL << NMI_SHIFT)

#define SOFTIRQ_DISABLE_OFFSET        (2 * SOFTIRQ_OFFSET)

#define PREEMPT_DISABLED        (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/*
 * Disable preemption until the scheduler is running -- use an unconditional
 * value so that it also works on !PREEMPT_COUNT kernels.
 *
 * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
 */
#define INIT_PREEMPT_COUNT        PREEMPT_OFFSET

/*
 * Initial preempt_count value; reflects the preempt_count schedule invariant
 * which states that during context switches:
 *
 *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
 *
 * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
 * Note: See finish_task_switch().
 */
#define FORK_PREEMPT_COUNT        (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)

/* preempt_count() and related functions, depends on PREEMPT_NEED_RESCHED */
#include <asm/preempt.h>

/**
 * interrupt_context_level - return interrupt context level
 *
 * Returns the current interrupt context level.
 *  0 - normal context
 *  1 - softirq context
 *  2 - hardirq context
 *  3 - NMI context
 */
static __always_inline unsigned char interrupt_context_level(void)
{
        unsigned long pc = preempt_count();
        unsigned char level = 0;

        level += !!(pc & (NMI_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK));
        level += !!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET));

        return level;
}

/*
 * These macro definitions avoid redundant invocations of preempt_count()
 * because such invocations would result in redundant loads given that
 * preempt_count() is commonly implemented with READ_ONCE().
 */

#define nmi_count()        (preempt_count() & NMI_MASK)
#define hardirq_count()        (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count()        (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count()                ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
#else
# define softirq_count()        (preempt_count() & SOFTIRQ_MASK)
# define irq_count()                (preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_MASK))
#endif

/*
 * Macros to retrieve the current execution context:
 *
 * in_nmi()                - We're in NMI context
 * in_hardirq()                - We're in hard IRQ context
 * in_serving_softirq()        - We're in softirq context
 * in_task()                - We're in task context
 */
#define in_nmi()                (nmi_count())
#define in_hardirq()                (hardirq_count())
#define in_serving_softirq()        (softirq_count() & SOFTIRQ_OFFSET)
#ifdef CONFIG_PREEMPT_RT
# define in_task()                (!((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | in_serving_softirq()))
#else
# define in_task()                (!(preempt_count() & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
#endif

/*
 * The following macros are deprecated and should not be used in new code:
 * in_irq()       - Obsolete version of in_hardirq()
 * in_softirq()   - We have BH disabled, or are processing softirqs
 * in_interrupt() - We're in NMI,IRQ,SoftIRQ context or have BH disabled
 */
#define in_irq()                (hardirq_count())
#define in_softirq()                (softirq_count())
#define in_interrupt()                (irq_count())

/*
 * The preempt_count offset after preempt_disable();
 */
#if defined(CONFIG_PREEMPT_COUNT)
# define PREEMPT_DISABLE_OFFSET        PREEMPT_OFFSET
#else
# define PREEMPT_DISABLE_OFFSET        0
#endif

/*
 * The preempt_count offset after spin_lock()
 */
#if !defined(CONFIG_PREEMPT_RT)
#define PREEMPT_LOCK_OFFSET                PREEMPT_DISABLE_OFFSET
#else
/* Locks on RT do not disable preemption */
#define PREEMPT_LOCK_OFFSET                0
#endif

/*
 * The preempt_count offset needed for things like:
 *
 *  spin_lock_bh()
 *
 * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and
 * softirqs, such that unlock sequences of:
 *
 *  spin_unlock();
 *  local_bh_enable();
 *
 * Work as expected.
 */
#define SOFTIRQ_LOCK_OFFSET (SOFTIRQ_DISABLE_OFFSET + PREEMPT_LOCK_OFFSET)

/*
 * Are we running in atomic context?  WARNING: this macro cannot
 * always detect atomic context; in particular, it cannot know about
 * held spinlocks in non-preemptible kernels.  Thus it should not be
 * used in the general case to determine whether sleeping is possible.
 * Do not use in_atomic() in driver code.
 */
#define in_atomic()        (preempt_count() != 0)

/*
 * Check whether we were atomic before we did preempt_disable():
 * (used by the scheduler)
 */
#define in_atomic_preempt_off() (preempt_count() != PREEMPT_DISABLE_OFFSET)

#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
extern void preempt_count_add(int val);
extern void preempt_count_sub(int val);
#define preempt_count_dec_and_test() \
        ({ preempt_count_sub(1); should_resched(0); })
#else
#define preempt_count_add(val)        __preempt_count_add(val)
#define preempt_count_sub(val)        __preempt_count_sub(val)
#define preempt_count_dec_and_test() __preempt_count_dec_and_test()
#endif

#define __preempt_count_inc() __preempt_count_add(1)
#define __preempt_count_dec() __preempt_count_sub(1)

#define preempt_count_inc() preempt_count_add(1)
#define preempt_count_dec() preempt_count_sub(1)

#ifdef CONFIG_PREEMPT_COUNT

#define preempt_disable() \
do { \
        preempt_count_inc(); \
        barrier(); \
} while (0)

#define sched_preempt_enable_no_resched() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_no_resched() sched_preempt_enable_no_resched()

#define preemptible()        (preempt_count() == 0 && !irqs_disabled())

#ifdef CONFIG_PREEMPTION
#define preempt_enable() \
do { \
        barrier(); \
        if (unlikely(preempt_count_dec_and_test())) \
                __preempt_schedule(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        if (unlikely(__preempt_count_dec_and_test())) \
                __preempt_schedule_notrace(); \
} while (0)

#define preempt_check_resched() \
do { \
        if (should_resched(0)) \
                __preempt_schedule(); \
} while (0)

#else /* !CONFIG_PREEMPTION */
#define preempt_enable() \
do { \
        barrier(); \
        preempt_count_dec(); \
} while (0)

#define preempt_enable_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#define preempt_check_resched() do { } while (0)
#endif /* CONFIG_PREEMPTION */

#define preempt_disable_notrace() \
do { \
        __preempt_count_inc(); \
        barrier(); \
} while (0)

#define preempt_enable_no_resched_notrace() \
do { \
        barrier(); \
        __preempt_count_dec(); \
} while (0)

#else /* !CONFIG_PREEMPT_COUNT */

/*
 * Even if we don't have any preemption, we need preempt disable/enable
 * to be barriers, so that we don't have things like get_user/put_user
 * that can cause faults and scheduling migrate into our preempt-protected
 * region.
 */
#define preempt_disable()                        barrier()
#define sched_preempt_enable_no_resched()        barrier()
#define preempt_enable_no_resched()                barrier()
#define preempt_enable()                        barrier()
#define preempt_check_resched()                        do { } while (0)

#define preempt_disable_notrace()                barrier()
#define preempt_enable_no_resched_notrace()        barrier()
#define preempt_enable_notrace()                barrier()
#define preemptible()                                0

#endif /* CONFIG_PREEMPT_COUNT */

#ifdef MODULE
/*
 * Modules have no business playing preemption tricks.
 */
#undef sched_preempt_enable_no_resched
#undef preempt_enable_no_resched
#undef preempt_enable_no_resched_notrace
#undef preempt_check_resched
#endif

#define preempt_set_need_resched() \
do { \
        set_preempt_need_resched(); \
} while (0)
#define preempt_fold_need_resched() \
do { \
        if (tif_need_resched()) \
                set_preempt_need_resched(); \
} while (0)

#ifdef CONFIG_PREEMPT_NOTIFIERS

struct preempt_notifier;
struct task_struct;

/**
 * preempt_ops - notifiers called when a task is preempted and rescheduled
 * @sched_in: we're about to be rescheduled:
 *    notifier: struct preempt_notifier for the task being scheduled
 *    cpu:  cpu we're scheduled on
 * @sched_out: we've just been preempted
 *    notifier: struct preempt_notifier for the task being preempted
 *    next: the task that's kicking us out
 *
 * Please note that sched_in and out are called under different
 * contexts.  sched_out is called with rq lock held and irq disabled
 * while sched_in is called without rq lock and irq enabled.  This
 * difference is intentional and depended upon by its users.
 */
struct preempt_ops {
        void (*sched_in)(struct preempt_notifier *notifier, int cpu);
        void (*sched_out)(struct preempt_notifier *notifier,
                          struct task_struct *next);
};

/**
 * preempt_notifier - key for installing preemption notifiers
 * @link: internal use
 * @ops: defines the notifier functions to be called
 *
 * Usually used in conjunction with container_of().
 */
struct preempt_notifier {
        struct hlist_node link;
        struct preempt_ops *ops;
};

void preempt_notifier_inc(void);
void preempt_notifier_dec(void);
void preempt_notifier_register(struct preempt_notifier *notifier);
void preempt_notifier_unregister(struct preempt_notifier *notifier);

static inline void preempt_notifier_init(struct preempt_notifier *notifier,
                                     struct preempt_ops *ops)
{
        /* INIT_HLIST_NODE() open coded, to avoid dependency on list.h */
        notifier->link.next = NULL;
        notifier->link.pprev = NULL;
        notifier->ops = ops;
}

#endif

#ifdef CONFIG_SMP

/*
 * Migrate-Disable and why it is undesired.
 *
 * When a preempted task becomes elegible to run under the ideal model (IOW it
 * becomes one of the M highest priority tasks), it might still have to wait
 * for the preemptee's migrate_disable() section to complete. Thereby suffering
 * a reduction in bandwidth in the exact duration of the migrate_disable()
 * section.
 *
 * Per this argument, the change from preempt_disable() to migrate_disable()
 * gets us:
 *
 * - a higher priority tasks gains reduced wake-up latency; with preempt_disable()
 *   it would have had to wait for the lower priority task.
 *
 * - a lower priority tasks; which under preempt_disable() could've instantly
 *   migrated away when another CPU becomes available, is now constrained
 *   by the ability to push the higher priority task away, which might itself be
 *   in a migrate_disable() section, reducing it's available bandwidth.
 *
 * IOW it trades latency / moves the interference term, but it stays in the
 * system, and as long as it remains unbounded, the system is not fully
 * deterministic.
 *
 *
 * The reason we have it anyway.
 *
 * PREEMPT_RT breaks a number of assumptions traditionally held. By forcing a
 * number of primitives into becoming preemptible, they would also allow
 * migration. This turns out to break a bunch of per-cpu usage. To this end,
 * all these primitives employ migirate_disable() to restore this implicit
 * assumption.
 *
 * This is a 'temporary' work-around at best. The correct solution is getting
 * rid of the above assumptions and reworking the code to employ explicit
 * per-cpu locking or short preempt-disable regions.
 *
 * The end goal must be to get rid of migrate_disable(), alternatively we need
 * a schedulability theory that does not depend on abritrary migration.
 *
 *
 * Notes on the implementation.
 *
 * The implementation is particularly tricky since existing code patterns
 * dictate neither migrate_disable() nor migrate_enable() is allowed to block.
 * This means that it cannot use cpus_read_lock() to serialize against hotplug,
 * nor can it easily migrate itself into a pending affinity mask change on
 * migrate_enable().
 *
 *
 * Note: even non-work-conserving schedulers like semi-partitioned depends on
 *       migration, so migrate_disable() is not only a problem for
 *       work-conserving schedulers.
 *
 */
extern void migrate_disable(void);
extern void migrate_enable(void);

#else

static inline void migrate_disable(void) { }
static inline void migrate_enable(void) { }

#endif /* CONFIG_SMP */

/**
 * preempt_disable_nested - Disable preemption inside a normally preempt disabled section
 *
 * Use for code which requires preemption protection inside a critical
 * section which has preemption disabled implicitly on non-PREEMPT_RT
 * enabled kernels, by e.g.:
 *  - holding a spinlock/rwlock
 *  - soft interrupt context
 *  - regular interrupt handlers
 *
 * On PREEMPT_RT enabled kernels spinlock/rwlock held sections, soft
 * interrupt context and regular interrupt handlers are preemptible and
 * only prevent migration. preempt_disable_nested() ensures that preemption
 * is disabled for cases which require CPU local serialization even on
 * PREEMPT_RT. For non-PREEMPT_RT kernels this is a NOP.
 *
 * The use cases are code sequences which are not serialized by a
 * particular lock instance, e.g.:
 *  - seqcount write side critical sections where the seqcount is not
 *    associated to a particular lock and therefore the automatic
 *    protection mechanism does not work. This prevents a live lock
 *    against a preempting high priority reader.
 *  - RMW per CPU variable updates like vmstat.
 */
/* Macro to avoid header recursion hell vs. lockdep */
#define preempt_disable_nested()                                \
do {                                                                \
        if (IS_ENABLED(CONFIG_PREEMPT_RT))                        \
                preempt_disable();                                \
        else                                                        \
                lockdep_assert_preemption_disabled();                \
} while (0)

/**
 * preempt_enable_nested - Undo the effect of preempt_disable_nested()
 */
static __always_inline void preempt_enable_nested(void)
{
        if (IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable();
}

DEFINE_LOCK_GUARD_0(preempt, preempt_disable(), preempt_enable())
DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_notrace())
DEFINE_LOCK_GUARD_0(migrate, migrate_disable(), migrate_enable())

#ifdef CONFIG_PREEMPT_DYNAMIC

extern bool preempt_model_none(void);
extern bool preempt_model_voluntary(void);
extern bool preempt_model_full(void);
extern bool preempt_model_lazy(void);

#else

static inline bool preempt_model_none(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_NONE);
}
static inline bool preempt_model_voluntary(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
}
static inline bool preempt_model_full(void)
{
        return IS_ENABLED(CONFIG_PREEMPT);
}

static inline bool preempt_model_lazy(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_LAZY);
}

#endif

static inline bool preempt_model_rt(void)
{
        return IS_ENABLED(CONFIG_PREEMPT_RT);
}

extern const char *preempt_model_str(void);

/*
 * Does the preemption model allow non-cooperative preemption?
 *
 * For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
 * CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
 * kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
 * PREEMPT_NONE model.
 */
static inline bool preempt_model_preemptible(void)
{
        return preempt_model_full() || preempt_model_lazy() || preempt_model_rt();
}

#endif /* __LINUX_PREEMPT_H */











































































































































































































  740 














  740 






























  201 













  201 











































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Credentials management - see Documentation/security/credentials.rst
 *
 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#ifndef _LINUX_CRED_H
#define _LINUX_CRED_H

#include <linux/capability.h>
#include <linux/init.h>
#include <linux/key.h>
#include <linux/atomic.h>
#include <linux/refcount.h>
#include <linux/uidgid.h>
#include <linux/sched.h>
#include <linux/sched/user.h>

struct cred;
struct inode;

/*
 * COW Supplementary groups list
 */
struct group_info {
        refcount_t        usage;
        int                ngroups;
        kgid_t                gid[];
} __randomize_layout;

/**
 * get_group_info - Get a reference to a group info structure
 * @group_info: The group info to reference
 *
 * This gets a reference to a set of supplementary groups.
 *
 * If the caller is accessing a task's credentials, they must hold the RCU read
 * lock when reading.
 */
static inline struct group_info *get_group_info(struct group_info *gi)
{
        refcount_inc(&gi->usage);
        return gi;
}

/**
 * put_group_info - Release a reference to a group info structure
 * @group_info: The group info to release
 */
#define put_group_info(group_info)                        \
do {                                                        \
        if (refcount_dec_and_test(&(group_info)->usage))        \
                groups_free(group_info);                \
} while (0)

#ifdef CONFIG_MULTIUSER
extern struct group_info *groups_alloc(int);
extern void groups_free(struct group_info *);

extern int in_group_p(kgid_t);
extern int in_egroup_p(kgid_t);
extern int groups_search(const struct group_info *, kgid_t);

extern int set_current_groups(struct group_info *);
extern void set_groups(struct cred *, struct group_info *);
extern bool may_setgroups(void);
extern void groups_sort(struct group_info *);
#else
static inline void groups_free(struct group_info *group_info)
{
}

static inline int in_group_p(kgid_t grp)
{
        return 1;
}
static inline int in_egroup_p(kgid_t grp)
{
        return 1;
}
static inline int groups_search(const struct group_info *group_info, kgid_t grp)
{
        return 1;
}
#endif

/*
 * The security context of a task
 *
 * The parts of the context break down into two categories:
 *
 *  (1) The objective context of a task.  These parts are used when some other
 *        task is attempting to affect this one.
 *
 *  (2) The subjective context.  These details are used when the task is acting
 *        upon another object, be that a file, a task, a key or whatever.
 *
 * Note that some members of this structure belong to both categories - the
 * LSM security pointer for instance.
 *
 * A task has two security pointers.  task->real_cred points to the objective
 * context that defines that task's actual details.  The objective part of this
 * context is used whenever that task is acted upon.
 *
 * task->cred points to the subjective context that defines the details of how
 * that task is going to act upon another object.  This may be overridden
 * temporarily to point to another security context, but normally points to the
 * same context as task->real_cred.
 */
struct cred {
        atomic_long_t        usage;
        kuid_t                uid;                /* real UID of the task */
        kgid_t                gid;                /* real GID of the task */
        kuid_t                suid;                /* saved UID of the task */
        kgid_t                sgid;                /* saved GID of the task */
        kuid_t                euid;                /* effective UID of the task */
        kgid_t                egid;                /* effective GID of the task */
        kuid_t                fsuid;                /* UID for VFS ops */
        kgid_t                fsgid;                /* GID for VFS ops */
        unsigned        securebits;        /* SUID-less security management */
        kernel_cap_t        cap_inheritable; /* caps our children can inherit */
        kernel_cap_t        cap_permitted;        /* caps we're permitted */
        kernel_cap_t        cap_effective;        /* caps we can actually use */
        kernel_cap_t        cap_bset;        /* capability bounding set */
        kernel_cap_t        cap_ambient;        /* Ambient capability set */
#ifdef CONFIG_KEYS
        unsigned char        jit_keyring;        /* default keyring to attach requested
                                         * keys to */
        struct key        *session_keyring; /* keyring inherited over fork */
        struct key        *process_keyring; /* keyring private to this process */
        struct key        *thread_keyring; /* keyring private to this thread */
        struct key        *request_key_auth; /* assumed request_key authority */
#endif
#ifdef CONFIG_SECURITY
        void                *security;        /* LSM security */
#endif
        struct user_struct *user;        /* real user ID subscription */
        struct user_namespace *user_ns; /* user_ns the caps and keyrings are relative to. */
        struct ucounts *ucounts;
        struct group_info *group_info;        /* supplementary groups for euid/fsgid */
        /* RCU deletion */
        union {
                int non_rcu;                        /* Can we skip RCU deletion? */
                struct rcu_head        rcu;                /* RCU deletion hook */
        };
} __randomize_layout;

extern void __put_cred(struct cred *);
extern void exit_creds(struct task_struct *);
extern int copy_creds(struct task_struct *, unsigned long);
extern const struct cred *get_task_cred(struct task_struct *);
extern struct cred *cred_alloc_blank(void);
extern struct cred *prepare_creds(void);
extern struct cred *prepare_exec_creds(void);
extern int commit_creds(struct cred *);
extern void abort_creds(struct cred *);
extern struct cred *prepare_kernel_cred(struct task_struct *);
extern int set_security_override(struct cred *, u32);
extern int set_security_override_from_ctx(struct cred *, const char *);
extern int set_create_files_as(struct cred *, struct inode *);
extern int cred_fscmp(const struct cred *, const struct cred *);
extern void __init cred_init(void);
extern int set_cred_ucounts(struct cred *);

static inline bool cap_ambient_invariant_ok(const struct cred *cred)
{
        return cap_issubset(cred->cap_ambient,
                            cap_intersect(cred->cap_permitted,
                                          cred->cap_inheritable));
}

static inline const struct cred *override_creds(const struct cred *override_cred)
{
        return rcu_replace_pointer(current->cred, override_cred, 1);
}

static inline const struct cred *revert_creds(const struct cred *revert_cred)
{
        return rcu_replace_pointer(current->cred, revert_cred, 1);
}

/**
 * get_cred_many - Get references on a set of credentials
 * @cred: The credentials to reference
 * @nr: Number of references to acquire
 *
 * Get references on the specified set of credentials.  The caller must release
 * all acquired reference.  If %NULL is passed, it is returned with no action.
 *
 * This is used to deal with a committed set of credentials.  Although the
 * pointer is const, this will temporarily discard the const and increment the
 * usage count.  The purpose of this is to attempt to catch at compile time the
 * accidental alteration of a set of credentials that should be considered
 * immutable.
 */
static inline const struct cred *get_cred_many(const struct cred *cred, int nr)
{
        struct cred *nonconst_cred = (struct cred *) cred;
        if (!cred)
                return cred;
        nonconst_cred->non_rcu = 0;
        atomic_long_add(nr, &nonconst_cred->usage);
        return cred;
}

/*
 * get_cred - Get a reference on a set of credentials
 * @cred: The credentials to reference
 *
 * Get a reference on the specified set of credentials.  The caller must
 * release the reference.  If %NULL is passed, it is returned with no action.
 *
 * This is used to deal with a committed set of credentials.
 */
static inline const struct cred *get_cred(const struct cred *cred)
{
        return get_cred_many(cred, 1);
}

static inline const struct cred *get_cred_rcu(const struct cred *cred)
{
        struct cred *nonconst_cred = (struct cred *) cred;
        if (!cred)
                return NULL;
        if (!atomic_long_inc_not_zero(&nonconst_cred->usage))
                return NULL;
        nonconst_cred->non_rcu = 0;
        return cred;
}

/**
 * put_cred - Release a reference to a set of credentials
 * @cred: The credentials to release
 * @nr: Number of references to release
 *
 * Release a reference to a set of credentials, deleting them when the last ref
 * is released.  If %NULL is passed, nothing is done.
 *
 * This takes a const pointer to a set of credentials because the credentials
 * on task_struct are attached by const pointers to prevent accidental
 * alteration of otherwise immutable credential sets.
 */
static inline void put_cred_many(const struct cred *_cred, int nr)
{
        struct cred *cred = (struct cred *) _cred;

        if (cred) {
                if (atomic_long_sub_and_test(nr, &cred->usage))
                        __put_cred(cred);
        }
}

/*
 * put_cred - Release a reference to a set of credentials
 * @cred: The credentials to release
 *
 * Release a reference to a set of credentials, deleting them when the last ref
 * is released.  If %NULL is passed, nothing is done.
 */
static inline void put_cred(const struct cred *cred)
{
        put_cred_many(cred, 1);
}

/**
 * current_cred - Access the current task's subjective credentials
 *
 * Access the subjective credentials of the current task.  RCU-safe,
 * since nobody else can modify it.
 */
#define current_cred() \
        rcu_dereference_protected(current->cred, 1)

/**
 * current_real_cred - Access the current task's objective credentials
 *
 * Access the objective credentials of the current task.  RCU-safe,
 * since nobody else can modify it.
 */
#define current_real_cred() \
        rcu_dereference_protected(current->real_cred, 1)

/**
 * __task_cred - Access a task's objective credentials
 * @task: The task to query
 *
 * Access the objective credentials of a task.  The caller must hold the RCU
 * readlock.
 *
 * The result of this function should not be passed directly to get_cred();
 * rather get_task_cred() should be used instead.
 */
#define __task_cred(task)        \
        rcu_dereference((task)->real_cred)

/**
 * get_current_cred - Get the current task's subjective credentials
 *
 * Get the subjective credentials of the current task, pinning them so that
 * they can't go away.  Accessing the current task's credentials directly is
 * not permitted.
 */
#define get_current_cred()                                \
        (get_cred(current_cred()))

/**
 * get_current_user - Get the current task's user_struct
 *
 * Get the user record of the current task, pinning it so that it can't go
 * away.
 */
#define get_current_user()                                \
({                                                        \
        struct user_struct *__u;                        \
        const struct cred *__cred;                        \
        __cred = current_cred();                        \
        __u = get_uid(__cred->user);                        \
        __u;                                                \
})

/**
 * get_current_groups - Get the current task's supplementary group list
 *
 * Get the supplementary group list of the current task, pinning it so that it
 * can't go away.
 */
#define get_current_groups()                                \
({                                                        \
        struct group_info *__groups;                        \
        const struct cred *__cred;                        \
        __cred = current_cred();                        \
        __groups = get_group_info(__cred->group_info);        \
        __groups;                                        \
})

#define task_cred_xxx(task, xxx)                        \
({                                                        \
        __typeof__(((struct cred *)NULL)->xxx) ___val;        \
        rcu_read_lock();                                \
        ___val = __task_cred((task))->xxx;                \
        rcu_read_unlock();                                \
        ___val;                                                \
})

#define task_uid(task)                (task_cred_xxx((task), uid))
#define task_euid(task)                (task_cred_xxx((task), euid))
#define task_ucounts(task)        (task_cred_xxx((task), ucounts))

#define current_cred_xxx(xxx)                        \
({                                                \
        current_cred()->xxx;                        \
})

#define current_uid()                (current_cred_xxx(uid))
#define current_gid()                (current_cred_xxx(gid))
#define current_euid()                (current_cred_xxx(euid))
#define current_egid()                (current_cred_xxx(egid))
#define current_suid()                (current_cred_xxx(suid))
#define current_sgid()                (current_cred_xxx(sgid))
#define current_fsuid()         (current_cred_xxx(fsuid))
#define current_fsgid()         (current_cred_xxx(fsgid))
#define current_cap()                (current_cred_xxx(cap_effective))
#define current_user()                (current_cred_xxx(user))
#define current_ucounts()        (current_cred_xxx(ucounts))

extern struct user_namespace init_user_ns;
#ifdef CONFIG_USER_NS
#define current_user_ns()        (current_cred_xxx(user_ns))
#else
static inline struct user_namespace *current_user_ns(void)
{
        return &init_user_ns;
}
#endif


#define current_uid_gid(_uid, _gid)                \
do {                                                \
        const struct cred *__cred;                \
        __cred = current_cred();                \
        *(_uid) = __cred->uid;                        \
        *(_gid) = __cred->gid;                        \
} while(0)

#define current_euid_egid(_euid, _egid)                \
do {                                                \
        const struct cred *__cred;                \
        __cred = current_cred();                \
        *(_euid) = __cred->euid;                \
        *(_egid) = __cred->egid;                \
} while(0)

#define current_fsuid_fsgid(_fsuid, _fsgid)        \
do {                                                \
        const struct cred *__cred;                \
        __cred = current_cred();                \
        *(_fsuid) = __cred->fsuid;                \
        *(_fsgid) = __cred->fsgid;                \
} while(0)

#endif /* _LINUX_CRED_H */




























































































































































































   80 






   78 


   80 




    1 









    1 







   79 






   79 


   80 












































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
// SPDX-License-Identifier: GPL-2.0-or-later

#define pr_fmt(fmt) "ref_tracker: " fmt

#include <linux/export.h>
#include <linux/list_sort.h>
#include <linux/ref_tracker.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/stackdepot.h>

#define REF_TRACKER_STACK_ENTRIES 16
#define STACK_BUF_SIZE 1024

struct ref_tracker {
        struct list_head        head;   /* anchor into dir->list or dir->quarantine */
        bool                        dead;
        depot_stack_handle_t        alloc_stack_handle;
        depot_stack_handle_t        free_stack_handle;
};

struct ref_tracker_dir_stats {
        int total;
        int count;
        struct {
                depot_stack_handle_t stack_handle;
                unsigned int count;
        } stacks[];
};

static struct ref_tracker_dir_stats *
ref_tracker_get_stats(struct ref_tracker_dir *dir, unsigned int limit)
{
        struct ref_tracker_dir_stats *stats;
        struct ref_tracker *tracker;

        stats = kmalloc(struct_size(stats, stacks, limit),
                        GFP_NOWAIT | __GFP_NOWARN);
        if (!stats)
                return ERR_PTR(-ENOMEM);
        stats->total = 0;
        stats->count = 0;

        list_for_each_entry(tracker, &dir->list, head) {
                depot_stack_handle_t stack = tracker->alloc_stack_handle;
                int i;

                ++stats->total;
                for (i = 0; i < stats->count; ++i)
                        if (stats->stacks[i].stack_handle == stack)
                                break;
                if (i >= limit)
                        continue;
                if (i >= stats->count) {
                        stats->stacks[i].stack_handle = stack;
                        stats->stacks[i].count = 0;
                        ++stats->count;
                }
                ++stats->stacks[i].count;
        }

        return stats;
}

struct ostream {
        char *buf;
        int size, used;
};

#define pr_ostream(stream, fmt, args...) \
({ \
        struct ostream *_s = (stream); \
\
        if (!_s->buf) { \
                pr_err(fmt, ##args); \
        } else { \
                int ret, len = _s->size - _s->used; \
                ret = snprintf(_s->buf + _s->used, len, pr_fmt(fmt), ##args); \
                _s->used += min(ret, len); \
        } \
})

static void
__ref_tracker_dir_pr_ostream(struct ref_tracker_dir *dir,
                             unsigned int display_limit, struct ostream *s)
{
        struct ref_tracker_dir_stats *stats;
        unsigned int i = 0, skipped;
        depot_stack_handle_t stack;
        char *sbuf;

        lockdep_assert_held(&dir->lock);

        if (list_empty(&dir->list))
                return;

        stats = ref_tracker_get_stats(dir, display_limit);
        if (IS_ERR(stats)) {
                pr_ostream(s, "%s@%pK: couldn't get stats, error %pe\n",
                           dir->name, dir, stats);
                return;
        }

        sbuf = kmalloc(STACK_BUF_SIZE, GFP_NOWAIT | __GFP_NOWARN);

        for (i = 0, skipped = stats->total; i < stats->count; ++i) {
                stack = stats->stacks[i].stack_handle;
                if (sbuf && !stack_depot_snprint(stack, sbuf, STACK_BUF_SIZE, 4))
                        sbuf[0] = 0;
                pr_ostream(s, "%s@%pK has %d/%d users at\n%s\n", dir->name, dir,
                           stats->stacks[i].count, stats->total, sbuf);
                skipped -= stats->stacks[i].count;
        }

        if (skipped)
                pr_ostream(s, "%s@%pK skipped reports about %d/%d users.\n",
                           dir->name, dir, skipped, stats->total);

        kfree(sbuf);

        kfree(stats);
}

void ref_tracker_dir_print_locked(struct ref_tracker_dir *dir,
                                  unsigned int display_limit)
{
        struct ostream os = {};

        __ref_tracker_dir_pr_ostream(dir, display_limit, &os);
}
EXPORT_SYMBOL(ref_tracker_dir_print_locked);

void ref_tracker_dir_print(struct ref_tracker_dir *dir,
                           unsigned int display_limit)
{
        unsigned long flags;

        spin_lock_irqsave(&dir->lock, flags);
        ref_tracker_dir_print_locked(dir, display_limit);
        spin_unlock_irqrestore(&dir->lock, flags);
}
EXPORT_SYMBOL(ref_tracker_dir_print);

int ref_tracker_dir_snprint(struct ref_tracker_dir *dir, char *buf, size_t size)
{
        struct ostream os = { .buf = buf, .size = size };
        unsigned long flags;

        spin_lock_irqsave(&dir->lock, flags);
        __ref_tracker_dir_pr_ostream(dir, 16, &os);
        spin_unlock_irqrestore(&dir->lock, flags);

        return os.used;
}
EXPORT_SYMBOL(ref_tracker_dir_snprint);

void ref_tracker_dir_exit(struct ref_tracker_dir *dir)
{
        struct ref_tracker *tracker, *n;
        unsigned long flags;
        bool leak = false;

        dir->dead = true;
        spin_lock_irqsave(&dir->lock, flags);
        list_for_each_entry_safe(tracker, n, &dir->quarantine, head) {
                list_del(&tracker->head);
                kfree(tracker);
                dir->quarantine_avail++;
        }
        if (!list_empty(&dir->list)) {
                ref_tracker_dir_print_locked(dir, 16);
                leak = true;
                list_for_each_entry_safe(tracker, n, &dir->list, head) {
                        list_del(&tracker->head);
                        kfree(tracker);
                }
        }
        spin_unlock_irqrestore(&dir->lock, flags);
        WARN_ON_ONCE(leak);
        WARN_ON_ONCE(refcount_read(&dir->untracked) != 1);
        WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1);
}
EXPORT_SYMBOL(ref_tracker_dir_exit);

int ref_tracker_alloc(struct ref_tracker_dir *dir,
                      struct ref_tracker **trackerp,
                      gfp_t gfp)
{
        unsigned long entries[REF_TRACKER_STACK_ENTRIES];
        struct ref_tracker *tracker;
        unsigned int nr_entries;
        gfp_t gfp_mask = gfp | __GFP_NOWARN;
        unsigned long flags;

        WARN_ON_ONCE(dir->dead);

        if (!trackerp) {
                refcount_inc(&dir->no_tracker);
                return 0;
        }
        if (gfp & __GFP_DIRECT_RECLAIM)
                gfp_mask |= __GFP_NOFAIL;
        *trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask);
        if (unlikely(!tracker)) {
                pr_err_once("memory allocation failure, unreliable refcount tracker.\n");
                refcount_inc(&dir->untracked);
                return -ENOMEM;
        }
        nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
        tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp);

        spin_lock_irqsave(&dir->lock, flags);
        list_add(&tracker->head, &dir->list);
        spin_unlock_irqrestore(&dir->lock, flags);
        return 0;
}
EXPORT_SYMBOL_GPL(ref_tracker_alloc);

int ref_tracker_free(struct ref_tracker_dir *dir,
                     struct ref_tracker **trackerp)
{
        unsigned long entries[REF_TRACKER_STACK_ENTRIES];
        depot_stack_handle_t stack_handle;
        struct ref_tracker *tracker;
        unsigned int nr_entries;
        unsigned long flags;

        WARN_ON_ONCE(dir->dead);

        if (!trackerp) {
                refcount_dec(&dir->no_tracker);
                return 0;
        }
        tracker = *trackerp;
        if (!tracker) {
                refcount_dec(&dir->untracked);
                return -EEXIST;
        }
        nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
        stack_handle = stack_depot_save(entries, nr_entries,
                                        GFP_NOWAIT | __GFP_NOWARN);

        spin_lock_irqsave(&dir->lock, flags);
        if (tracker->dead) {
                pr_err("reference already released.\n");
                if (tracker->alloc_stack_handle) {
                        pr_err("allocated in:\n");
                        stack_depot_print(tracker->alloc_stack_handle);
                }
                if (tracker->free_stack_handle) {
                        pr_err("freed in:\n");
                        stack_depot_print(tracker->free_stack_handle);
                }
                spin_unlock_irqrestore(&dir->lock, flags);
                WARN_ON_ONCE(1);
                return -EINVAL;
        }
        tracker->dead = true;

        tracker->free_stack_handle = stack_handle;

        list_move_tail(&tracker->head, &dir->quarantine);
        if (!dir->quarantine_avail) {
                tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head);
                list_del(&tracker->head);
        } else {
                dir->quarantine_avail--;
                tracker = NULL;
        }
        spin_unlock_irqrestore(&dir->lock, flags);

        kfree(tracker);
        return 0;
}
EXPORT_SYMBOL_GPL(ref_tracker_free);


























































































    1 



    1 

    1 











































































































































































































































































    4 
    4 

    4 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/netdevice.h>
#include <net/netdev_lock.h>

#include "dev.h"

/**
 * dev_change_name() - change name of a device
 * @dev: device
 * @newname: name (or format string) must be at least IFNAMSIZ
 *
 * Change name of a device, can pass format strings "eth%d".
 * for wildcarding.
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_name(struct net_device *dev, const char *newname)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_change_name(dev, newname);
        netdev_unlock_ops(dev);

        return ret;
}

/**
 * dev_set_alias() - change ifalias of a device
 * @dev: device
 * @alias: name up to IFALIASZ
 * @len: limit of bytes to copy from info
 *
 * Set ifalias for a device.
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_set_alias(dev, alias, len);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_set_alias);

/**
 * dev_change_flags() - change device settings
 * @dev: device
 * @flags: device state flags
 * @extack: netlink extended ack
 *
 * Change settings on device based state flags. The flags are
 * in the userspace exported format.
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_flags(struct net_device *dev, unsigned int flags,
                     struct netlink_ext_ack *extack)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_change_flags(dev, flags, extack);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_change_flags);

/**
 * dev_set_group() - change group this device belongs to
 * @dev: device
 * @new_group: group this device should belong to
 */
void dev_set_group(struct net_device *dev, int new_group)
{
        netdev_lock_ops(dev);
        netif_set_group(dev, new_group);
        netdev_unlock_ops(dev);
}

int dev_set_mac_address_user(struct net_device *dev,
                             struct sockaddr_storage *ss,
                             struct netlink_ext_ack *extack)
{
        int ret;

        down_write(&dev_addr_sem);
        netdev_lock_ops(dev);
        ret = netif_set_mac_address(dev, ss, extack);
        netdev_unlock_ops(dev);
        up_write(&dev_addr_sem);

        return ret;
}
EXPORT_SYMBOL(dev_set_mac_address_user);

/**
 * dev_change_net_namespace() - move device to different nethost namespace
 * @dev: device
 * @net: network namespace
 * @pat: If not NULL name pattern to try if the current device name
 *       is already taken in the destination network namespace.
 *
 * This function shuts down a device interface and moves it
 * to a new network namespace. On success 0 is returned, on
 * a failure a netagive errno code is returned.
 *
 * Callers must hold the rtnl semaphore.
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_net_namespace(struct net_device *dev, struct net *net,
                             const char *pat)
{
        return __dev_change_net_namespace(dev, net, pat, 0, NULL);
}
EXPORT_SYMBOL_GPL(dev_change_net_namespace);

/**
 * dev_change_carrier() - change device carrier
 * @dev: device
 * @new_carrier: new value
 *
 * Change device carrier
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_carrier(struct net_device *dev, bool new_carrier)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_change_carrier(dev, new_carrier);
        netdev_unlock_ops(dev);

        return ret;
}

/**
 * dev_change_tx_queue_len() - change TX queue length of a netdevice
 * @dev: device
 * @new_len: new tx queue length
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_change_tx_queue_len(dev, new_len);
        netdev_unlock_ops(dev);

        return ret;
}

/**
 * dev_change_proto_down() - set carrier according to proto_down
 * @dev: device
 * @proto_down: new value
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_change_proto_down(struct net_device *dev, bool proto_down)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_change_proto_down(dev, proto_down);
        netdev_unlock_ops(dev);

        return ret;
}

/**
 * dev_open() - prepare an interface for use
 * @dev: device to open
 * @extack: netlink extended ack
 *
 * Takes a device from down to up state. The device's private open
 * function is invoked and then the multicast lists are loaded. Finally
 * the device is moved into the up state and a %NETDEV_UP message is
 * sent to the netdev notifier chain.
 *
 * Calling this function on an active interface is a nop. On a failure
 * a negative errno code is returned.
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_open(dev, extack);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_open);

/**
 * dev_close() - shutdown an interface
 * @dev: device to shutdown
 *
 * This function moves an active device into down state. A
 * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
 * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
 * chain.
 */
void dev_close(struct net_device *dev)
{
        netdev_lock_ops(dev);
        netif_close(dev);
        netdev_unlock_ops(dev);
}
EXPORT_SYMBOL(dev_close);

int dev_eth_ioctl(struct net_device *dev,
                  struct ifreq *ifr, unsigned int cmd)
{
        const struct net_device_ops *ops = dev->netdev_ops;
        int ret = -ENODEV;

        if (!ops->ndo_eth_ioctl)
                return -EOPNOTSUPP;

        netdev_lock_ops(dev);
        if (netif_device_present(dev))
                ret = ops->ndo_eth_ioctl(dev, ifr, cmd);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_eth_ioctl);

int dev_set_mtu(struct net_device *dev, int new_mtu)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_set_mtu(dev, new_mtu);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_set_mtu);

/**
 * dev_disable_lro() - disable Large Receive Offload on a device
 * @dev: device
 *
 * Disable Large Receive Offload (LRO) on a net device.  Must be
 * called under RTNL.  This is needed if received packets may be
 * forwarded to another interface.
 */
void dev_disable_lro(struct net_device *dev)
{
        netdev_lock_ops(dev);
        netif_disable_lro(dev);
        netdev_unlock_ops(dev);
}
EXPORT_SYMBOL(dev_disable_lro);

/**
 * dev_set_promiscuity() - update promiscuity count on a device
 * @dev: device
 * @inc: modifier
 *
 * Add or remove promiscuity from a device. While the count in the device
 * remains above zero the interface remains promiscuous. Once it hits zero
 * the device reverts back to normal filtering operation. A negative inc
 * value is used to drop promiscuity on the device.
 * Return 0 if successful or a negative errno code on error.
 */
int dev_set_promiscuity(struct net_device *dev, int inc)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_set_promiscuity(dev, inc);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_set_promiscuity);

/**
 * dev_set_allmulti() - update allmulti count on a device
 * @dev: device
 * @inc: modifier
 *
 * Add or remove reception of all multicast frames to a device. While the
 * count in the device remains above zero the interface remains listening
 * to all interfaces. Once it hits zero the device reverts back to normal
 * filtering operation. A negative @inc value is used to drop the counter
 * when releasing a resource needing all multicasts.
 *
 * Return: 0 on success, -errno on failure.
 */

int dev_set_allmulti(struct net_device *dev, int inc)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_set_allmulti(dev, inc, true);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_set_allmulti);

/**
 * dev_set_mac_address() - change Media Access Control Address
 * @dev: device
 * @ss: new address
 * @extack: netlink extended ack
 *
 * Change the hardware (MAC) address of the device
 *
 * Return: 0 on success, -errno on failure.
 */
int dev_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
                        struct netlink_ext_ack *extack)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_set_mac_address(dev, ss, extack);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL(dev_set_mac_address);

int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
{
        int ret;

        netdev_lock_ops(dev);
        ret = netif_xdp_propagate(dev, bpf);
        netdev_unlock_ops(dev);

        return ret;
}
EXPORT_SYMBOL_GPL(dev_xdp_propagate);

/**
 * netdev_state_change() - device changes state
 * @dev: device to cause notification
 *
 * Called to indicate a device has changed state. This function calls
 * the notifier chains for netdev_chain and sends a NEWLINK message
 * to the routing socket.
 */
void netdev_state_change(struct net_device *dev)
{
        netdev_lock_ops(dev);
        netif_state_change(dev);
        netdev_unlock_ops(dev);
}
EXPORT_SYMBOL(netdev_state_change);

































































































































































































































































































































































































































































































































































































































































































  288 



  287 














































































































  305 






  305 




















































































  308 



  305 





  173 


  259 




























































































































































































































   14 

    3 





















































































































  304 



  305 

  112 


























  304 
    1 






























































































































































































































































































  302 



    1 

































































  303 
  304 





































































    1 














  304 


  303 


  305 






















    1 































  304 
























































































































































































































































































    4 















    4 



















































































    4 



    4 





















  304 






  305 
  305 
  305 




























    1 



















  304 











  305 






  305 




  305 



  304 
    1 




  305 












  305 


























  304 











  305 













  139 
  303 











  304 

































   12 



   13 

   13 

   13 



   13 
































































































  328 



  327 
  328 
  328 
  325 








  299 



   49 












   49 
























  323 





  324 

  323 

  325 



  325 
























    4 






    4 

    4 


















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 






    4 
    4 

    4 


    4 
































































































































   31 





   31 













   31 






   31 

   29 

    4 





   31 






   31 

   31 
   29 





   31 




   30 





   30 



























   30 

























   31 





   30 



    4 


   29 









































    4 
    4 





    4 






    4 






    4 

















    4 



























































































   29 


   27 










   14 





























   14 




   29 








   29 






















































   29 


















   29 
   29 
   29 










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
7663
7664
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
// SPDX-License-Identifier: GPL-2.0-only
/*
 * kernel/workqueue.c - generic async execution with shared worker pool
 *
 * Copyright (C) 2002                Ingo Molnar
 *
 *   Derived from the taskqueue/keventd code by:
 *     David Woodhouse <dwmw2@infradead.org>
 *     Andrew Morton
 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
 *     Theodore Ts'o <tytso@mit.edu>
 *
 * Made to use alloc_percpu by Christoph Lameter.
 *
 * Copyright (C) 2010                SUSE Linux Products GmbH
 * Copyright (C) 2010                Tejun Heo <tj@kernel.org>
 *
 * This is the generic async execution mechanism.  Work items as are
 * executed in process context.  The worker pool is shared and
 * automatically managed.  There are two worker pools for each CPU (one for
 * normal work items and the other for high priority ones) and some extra
 * pools for workqueues which are not bound to any specific CPU - the
 * number of these backing pools is dynamic.
 *
 * Please read Documentation/core-api/workqueue.rst for details.
 */

#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/signal.h>
#include <linux/completion.h>
#include <linux/workqueue.h>
#include <linux/slab.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/kthread.h>
#include <linux/hardirq.h>
#include <linux/mempolicy.h>
#include <linux/freezer.h>
#include <linux/debug_locks.h>
#include <linux/lockdep.h>
#include <linux/idr.h>
#include <linux/jhash.h>
#include <linux/hashtable.h>
#include <linux/rculist.h>
#include <linux/nodemask.h>
#include <linux/moduleparam.h>
#include <linux/uaccess.h>
#include <linux/sched/isolation.h>
#include <linux/sched/debug.h>
#include <linux/nmi.h>
#include <linux/kvm_para.h>
#include <linux/delay.h>
#include <linux/irq_work.h>

#include "workqueue_internal.h"

enum worker_pool_flags {
        /*
         * worker_pool flags
         *
         * A bound pool is either associated or disassociated with its CPU.
         * While associated (!DISASSOCIATED), all workers are bound to the
         * CPU and none has %WORKER_UNBOUND set and concurrency management
         * is in effect.
         *
         * While DISASSOCIATED, the cpu may be offline and all workers have
         * %WORKER_UNBOUND set and concurrency management disabled, and may
         * be executing on any CPU.  The pool behaves as an unbound one.
         *
         * Note that DISASSOCIATED should be flipped only while holding
         * wq_pool_attach_mutex to avoid changing binding state while
         * worker_attach_to_pool() is in progress.
         *
         * As there can only be one concurrent BH execution context per CPU, a
         * BH pool is per-CPU and always DISASSOCIATED.
         */
        POOL_BH                        = 1 << 0,        /* is a BH pool */
        POOL_MANAGER_ACTIVE        = 1 << 1,        /* being managed */
        POOL_DISASSOCIATED        = 1 << 2,        /* cpu can't serve workers */
        POOL_BH_DRAINING        = 1 << 3,        /* draining after CPU offline */
};

enum worker_flags {
        /* worker flags */
        WORKER_DIE                = 1 << 1,        /* die die die */
        WORKER_IDLE                = 1 << 2,        /* is idle */
        WORKER_PREP                = 1 << 3,        /* preparing to run works */
        WORKER_CPU_INTENSIVE        = 1 << 6,        /* cpu intensive */
        WORKER_UNBOUND                = 1 << 7,        /* worker is unbound */
        WORKER_REBOUND                = 1 << 8,        /* worker was rebound */

        WORKER_NOT_RUNNING        = WORKER_PREP | WORKER_CPU_INTENSIVE |
                                  WORKER_UNBOUND | WORKER_REBOUND,
};

enum work_cancel_flags {
        WORK_CANCEL_DELAYED        = 1 << 0,        /* canceling a delayed_work */
        WORK_CANCEL_DISABLE        = 1 << 1,        /* canceling to disable */
};

enum wq_internal_consts {
        NR_STD_WORKER_POOLS        = 2,                /* # standard pools per cpu */

        UNBOUND_POOL_HASH_ORDER        = 6,                /* hashed by pool->attrs */
        BUSY_WORKER_HASH_ORDER        = 6,                /* 64 pointers */

        MAX_IDLE_WORKERS_RATIO        = 4,                /* 1/4 of busy can be idle */
        IDLE_WORKER_TIMEOUT        = 300 * HZ,        /* keep idle ones for 5 mins */

        MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
                                                /* call for help after 10ms
                                                   (min two ticks) */
        MAYDAY_INTERVAL                = HZ / 10,        /* and then every 100ms */
        CREATE_COOLDOWN                = HZ,                /* time to breath after fail */

        /*
         * Rescue workers are used only on emergencies and shared by
         * all cpus.  Give MIN_NICE.
         */
        RESCUER_NICE_LEVEL        = MIN_NICE,
        HIGHPRI_NICE_LEVEL        = MIN_NICE,

        WQ_NAME_LEN                = 32,
        WORKER_ID_LEN                = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */
};

/*
 * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and
 * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because
 * msecs_to_jiffies() can't be an initializer.
 */
#define BH_WORKER_JIFFIES        msecs_to_jiffies(2)
#define BH_WORKER_RESTARTS        10

/*
 * Structure fields follow one of the following exclusion rules.
 *
 * I: Modifiable by initialization/destruction paths and read-only for
 *    everyone else.
 *
 * P: Preemption protected.  Disabling preemption is enough and should
 *    only be modified and accessed from the local cpu.
 *
 * L: pool->lock protected.  Access with pool->lock held.
 *
 * LN: pool->lock and wq_node_nr_active->lock protected for writes. Either for
 *     reads.
 *
 * K: Only modified by worker while holding pool->lock. Can be safely read by
 *    self, while holding pool->lock or from IRQ context if %current is the
 *    kworker.
 *
 * S: Only modified by worker self.
 *
 * A: wq_pool_attach_mutex protected.
 *
 * PL: wq_pool_mutex protected.
 *
 * PR: wq_pool_mutex protected for writes.  RCU protected for reads.
 *
 * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
 *
 * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
 *      RCU for reads.
 *
 * WQ: wq->mutex protected.
 *
 * WR: wq->mutex protected for writes.  RCU protected for reads.
 *
 * WO: wq->mutex protected for writes. Updated with WRITE_ONCE() and can be read
 *     with READ_ONCE() without locking.
 *
 * MD: wq_mayday_lock protected.
 *
 * WD: Used internally by the watchdog.
 */

/* struct worker is defined in workqueue_internal.h */

struct worker_pool {
        raw_spinlock_t                lock;                /* the pool lock */
        int                        cpu;                /* I: the associated cpu */
        int                        node;                /* I: the associated node ID */
        int                        id;                /* I: pool ID */
        unsigned int                flags;                /* L: flags */

        unsigned long                watchdog_ts;        /* L: watchdog timestamp */
        bool                        cpu_stall;        /* WD: stalled cpu bound pool */

        /*
         * The counter is incremented in a process context on the associated CPU
         * w/ preemption disabled, and decremented or reset in the same context
         * but w/ pool->lock held. The readers grab pool->lock and are
         * guaranteed to see if the counter reached zero.
         */
        int                        nr_running;

        struct list_head        worklist;        /* L: list of pending works */

        int                        nr_workers;        /* L: total number of workers */
        int                        nr_idle;        /* L: currently idle workers */

        struct list_head        idle_list;        /* L: list of idle workers */
        struct timer_list        idle_timer;        /* L: worker idle timeout */
        struct work_struct      idle_cull_work; /* L: worker idle cleanup */

        struct timer_list        mayday_timer;          /* L: SOS timer for workers */

        /* a workers is either on busy_hash or idle_list, or the manager */
        DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
                                                /* L: hash of busy workers */

        struct worker                *manager;        /* L: purely informational */
        struct list_head        workers;        /* A: attached workers */

        struct ida                worker_ida;        /* worker IDs for task name */

        struct workqueue_attrs        *attrs;                /* I: worker attributes */
        struct hlist_node        hash_node;        /* PL: unbound_pool_hash node */
        int                        refcnt;                /* PL: refcnt for unbound pools */

        /*
         * Destruction of pool is RCU protected to allow dereferences
         * from get_work_pool().
         */
        struct rcu_head                rcu;
};

/*
 * Per-pool_workqueue statistics. These can be monitored using
 * tools/workqueue/wq_monitor.py.
 */
enum pool_workqueue_stats {
        PWQ_STAT_STARTED,        /* work items started execution */
        PWQ_STAT_COMPLETED,        /* work items completed execution */
        PWQ_STAT_CPU_TIME,        /* total CPU time consumed */
        PWQ_STAT_CPU_INTENSIVE,        /* wq_cpu_intensive_thresh_us violations */
        PWQ_STAT_CM_WAKEUP,        /* concurrency-management worker wakeups */
        PWQ_STAT_REPATRIATED,        /* unbound workers brought back into scope */
        PWQ_STAT_MAYDAY,        /* maydays to rescuer */
        PWQ_STAT_RESCUED,        /* linked work items executed by rescuer */

        PWQ_NR_STATS,
};

/*
 * The per-pool workqueue.  While queued, bits below WORK_PWQ_SHIFT
 * of work_struct->data are used for flags and the remaining high bits
 * point to the pwq; thus, pwqs need to be aligned at two's power of the
 * number of flag bits.
 */
struct pool_workqueue {
        struct worker_pool        *pool;                /* I: the associated pool */
        struct workqueue_struct *wq;                /* I: the owning workqueue */
        int                        work_color;        /* L: current color */
        int                        flush_color;        /* L: flushing color */
        int                        refcnt;                /* L: reference count */
        int                        nr_in_flight[WORK_NR_COLORS];
                                                /* L: nr of in_flight works */
        bool                        plugged;        /* L: execution suspended */

        /*
         * nr_active management and WORK_STRUCT_INACTIVE:
         *
         * When pwq->nr_active >= max_active, new work item is queued to
         * pwq->inactive_works instead of pool->worklist and marked with
         * WORK_STRUCT_INACTIVE.
         *
         * All work items marked with WORK_STRUCT_INACTIVE do not participate in
         * nr_active and all work items in pwq->inactive_works are marked with
         * WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE work items are
         * in pwq->inactive_works. Some of them are ready to run in
         * pool->worklist or worker->scheduled. Those work itmes are only struct
         * wq_barrier which is used for flush_work() and should not participate
         * in nr_active. For non-barrier work item, it is marked with
         * WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
         */
        int                        nr_active;        /* L: nr of active works */
        struct list_head        inactive_works;        /* L: inactive works */
        struct list_head        pending_node;        /* LN: node on wq_node_nr_active->pending_pwqs */
        struct list_head        pwqs_node;        /* WR: node on wq->pwqs */
        struct list_head        mayday_node;        /* MD: node on wq->maydays */

        u64                        stats[PWQ_NR_STATS];

        /*
         * Release of unbound pwq is punted to a kthread_worker. See put_pwq()
         * and pwq_release_workfn() for details. pool_workqueue itself is also
         * RCU protected so that the first pwq can be determined without
         * grabbing wq->mutex.
         */
        struct kthread_work        release_work;
        struct rcu_head                rcu;
} __aligned(1 << WORK_STRUCT_PWQ_SHIFT);

/*
 * Structure used to wait for workqueue flush.
 */
struct wq_flusher {
        struct list_head        list;                /* WQ: list of flushers */
        int                        flush_color;        /* WQ: flush color waiting for */
        struct completion        done;                /* flush completion */
};

struct wq_device;

/*
 * Unlike in a per-cpu workqueue where max_active limits its concurrency level
 * on each CPU, in an unbound workqueue, max_active applies to the whole system.
 * As sharing a single nr_active across multiple sockets can be very expensive,
 * the counting and enforcement is per NUMA node.
 *
 * The following struct is used to enforce per-node max_active. When a pwq wants
 * to start executing a work item, it should increment ->nr using
 * tryinc_node_nr_active(). If acquisition fails due to ->nr already being over
 * ->max, the pwq is queued on ->pending_pwqs. As in-flight work items finish
 * and decrement ->nr, node_activate_pending_pwq() activates the pending pwqs in
 * round-robin order.
 */
struct wq_node_nr_active {
        int                        max;                /* per-node max_active */
        atomic_t                nr;                /* per-node nr_active */
        raw_spinlock_t                lock;                /* nests inside pool locks */
        struct list_head        pending_pwqs;        /* LN: pwqs with inactive works */
};

/*
 * The externally visible workqueue.  It relays the issued work items to
 * the appropriate worker_pool through its pool_workqueues.
 */
struct workqueue_struct {
        struct list_head        pwqs;                /* WR: all pwqs of this wq */
        struct list_head        list;                /* PR: list of all workqueues */

        struct mutex                mutex;                /* protects this wq */
        int                        work_color;        /* WQ: current work color */
        int                        flush_color;        /* WQ: current flush color */
        atomic_t                nr_pwqs_to_flush; /* flush in progress */
        struct wq_flusher        *first_flusher;        /* WQ: first flusher */
        struct list_head        flusher_queue;        /* WQ: flush waiters */
        struct list_head        flusher_overflow; /* WQ: flush overflow list */

        struct list_head        maydays;        /* MD: pwqs requesting rescue */
        struct worker                *rescuer;        /* MD: rescue worker */

        int                        nr_drainers;        /* WQ: drain in progress */

        /* See alloc_workqueue() function comment for info on min/max_active */
        int                        max_active;        /* WO: max active works */
        int                        min_active;        /* WO: min active works */
        int                        saved_max_active; /* WQ: saved max_active */
        int                        saved_min_active; /* WQ: saved min_active */

        struct workqueue_attrs        *unbound_attrs;        /* PW: only for unbound wqs */
        struct pool_workqueue __rcu *dfl_pwq;   /* PW: only for unbound wqs */

#ifdef CONFIG_SYSFS
        struct wq_device        *wq_dev;        /* I: for sysfs interface */
#endif
#ifdef CONFIG_LOCKDEP
        char                        *lock_name;
        struct lock_class_key        key;
        struct lockdep_map        __lockdep_map;
        struct lockdep_map        *lockdep_map;
#endif
        char                        name[WQ_NAME_LEN]; /* I: workqueue name */

        /*
         * Destruction of workqueue_struct is RCU protected to allow walking
         * the workqueues list without grabbing wq_pool_mutex.
         * This is used to dump all workqueues from sysrq.
         */
        struct rcu_head                rcu;

        /* hot fields used during command issue, aligned to cacheline */
        unsigned int                flags ____cacheline_aligned; /* WQ: WQ_* flags */
        struct pool_workqueue __rcu * __percpu *cpu_pwq; /* I: per-cpu pwqs */
        struct wq_node_nr_active *node_nr_active[]; /* I: per-node nr_active */
};

/*
 * Each pod type describes how CPUs should be grouped for unbound workqueues.
 * See the comment above workqueue_attrs->affn_scope.
 */
struct wq_pod_type {
        int                        nr_pods;        /* number of pods */
        cpumask_var_t                *pod_cpus;        /* pod -> cpus */
        int                        *pod_node;        /* pod -> node */
        int                        *cpu_pod;        /* cpu -> pod */
};

struct work_offq_data {
        u32                        pool_id;
        u32                        disable;
        u32                        flags;
};

static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = {
        [WQ_AFFN_DFL]                = "default",
        [WQ_AFFN_CPU]                = "cpu",
        [WQ_AFFN_SMT]                = "smt",
        [WQ_AFFN_CACHE]                = "cache",
        [WQ_AFFN_NUMA]                = "numa",
        [WQ_AFFN_SYSTEM]        = "system",
};

/*
 * Per-cpu work items which run for longer than the following threshold are
 * automatically considered CPU intensive and excluded from concurrency
 * management to prevent them from noticeably delaying other per-cpu work items.
 * ULONG_MAX indicates that the user hasn't overridden it with a boot parameter.
 * The actual value is initialized in wq_cpu_intensive_thresh_init().
 */
static unsigned long wq_cpu_intensive_thresh_us = ULONG_MAX;
module_param_named(cpu_intensive_thresh_us, wq_cpu_intensive_thresh_us, ulong, 0644);
#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT
static unsigned int wq_cpu_intensive_warning_thresh = 4;
module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh, uint, 0644);
#endif

/* see the comment above the definition of WQ_POWER_EFFICIENT */
static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
module_param_named(power_efficient, wq_power_efficient, bool, 0444);

static bool wq_online;                        /* can kworkers be created yet? */
static bool wq_topo_initialized __read_mostly = false;

static struct kmem_cache *pwq_cache;

static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES];
static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE;

/* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */
static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf;

static DEFINE_MUTEX(wq_pool_mutex);        /* protects pools and workqueues list */
static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
static DEFINE_RAW_SPINLOCK(wq_mayday_lock);        /* protects wq->maydays list */
/* wait for manager to go away */
static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);

static LIST_HEAD(workqueues);                /* PR: list of all workqueues */
static bool workqueue_freezing;                /* PL: have wqs started freezing? */

/* PL: mirror the cpu_online_mask excluding the CPU in the midst of hotplugging */
static cpumask_var_t wq_online_cpumask;

/* PL&A: allowable cpus for unbound wqs and work items */
static cpumask_var_t wq_unbound_cpumask;

/* PL: user requested unbound cpumask via sysfs */
static cpumask_var_t wq_requested_unbound_cpumask;

/* PL: isolated cpumask to be excluded from unbound cpumask */
static cpumask_var_t wq_isolated_cpumask;

/* for further constrain wq_unbound_cpumask by cmdline parameter*/
static struct cpumask wq_cmdline_cpumask __initdata;

/* CPU where unbound work was last round robin scheduled from this CPU */
static DEFINE_PER_CPU(int, wq_rr_cpu_last);

/*
 * Local execution of unbound work items is no longer guaranteed.  The
 * following always forces round-robin CPU selection on unbound work items
 * to uncover usages which depend on it.
 */
#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
static bool wq_debug_force_rr_cpu = true;
#else
static bool wq_debug_force_rr_cpu = false;
#endif
module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);

/* to raise softirq for the BH worker pools on other CPUs */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_work [NR_STD_WORKER_POOLS], bh_pool_irq_works);

/* the BH worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], bh_worker_pools);

/* the per-cpu worker pools */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);

static DEFINE_IDR(worker_pool_idr);        /* PR: idr of all pools */

/* PL: hash of all unbound pools keyed by pool->attrs */
static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);

/* I: attributes used when instantiating standard unbound pools on demand */
static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];

/* I: attributes used when instantiating ordered pools on demand */
static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];

/*
 * I: kthread_worker to release pwq's. pwq release needs to be bounced to a
 * process context while holding a pool lock. Bounce to a dedicated kthread
 * worker to avoid A-A deadlocks.
 */
static struct kthread_worker *pwq_release_worker __ro_after_init;

struct workqueue_struct *system_wq __ro_after_init;
EXPORT_SYMBOL(system_wq);
struct workqueue_struct *system_highpri_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_highpri_wq);
struct workqueue_struct *system_long_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_long_wq);
struct workqueue_struct *system_unbound_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_unbound_wq);
struct workqueue_struct *system_freezable_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_wq);
struct workqueue_struct *system_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_power_efficient_wq);
struct workqueue_struct *system_freezable_power_efficient_wq __ro_after_init;
EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
struct workqueue_struct *system_bh_wq;
EXPORT_SYMBOL_GPL(system_bh_wq);
struct workqueue_struct *system_bh_highpri_wq;
EXPORT_SYMBOL_GPL(system_bh_highpri_wq);

static int worker_thread(void *__worker);
static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
static void show_pwq(struct pool_workqueue *pwq);
static void show_one_worker_pool(struct worker_pool *pool);

#define CREATE_TRACE_POINTS
#include <trace/events/workqueue.h>

#define assert_rcu_or_pool_mutex()                                        \
        RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&                        \
                         !lockdep_is_held(&wq_pool_mutex),                \
                         "RCU or wq_pool_mutex should be held")

#define assert_rcu_or_wq_mutex_or_pool_mutex(wq)                        \
        RCU_LOCKDEP_WARN(!rcu_read_lock_any_held() &&                        \
                         !lockdep_is_held(&wq->mutex) &&                \
                         !lockdep_is_held(&wq_pool_mutex),                \
                         "RCU, wq->mutex or wq_pool_mutex should be held")

#define for_each_bh_worker_pool(pool, cpu)                                \
        for ((pool) = &per_cpu(bh_worker_pools, cpu)[0];                \
             (pool) < &per_cpu(bh_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
             (pool)++)

#define for_each_cpu_worker_pool(pool, cpu)                                \
        for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];                \
             (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
             (pool)++)

/**
 * for_each_pool - iterate through all worker_pools in the system
 * @pool: iteration cursor
 * @pi: integer used for iteration
 *
 * This must be called either with wq_pool_mutex held or RCU read
 * locked.  If the pool needs to be used beyond the locking in effect, the
 * caller is responsible for guaranteeing that the pool stays online.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pool(pool, pi)                                                \
        idr_for_each_entry(&worker_pool_idr, pool, pi)                        \
                if (({ assert_rcu_or_pool_mutex(); false; })) { }        \
                else

/**
 * for_each_pool_worker - iterate through all workers of a worker_pool
 * @worker: iteration cursor
 * @pool: worker_pool to iterate workers of
 *
 * This must be called with wq_pool_attach_mutex.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pool_worker(worker, pool)                                \
        list_for_each_entry((worker), &(pool)->workers, node)                \
                if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
                else

/**
 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
 * @pwq: iteration cursor
 * @wq: the target workqueue
 *
 * This must be called either with wq->mutex held or RCU read locked.
 * If the pwq needs to be used beyond the locking in effect, the caller is
 * responsible for guaranteeing that the pwq stays online.
 *
 * The if/else clause exists only for the lockdep assertion and can be
 * ignored.
 */
#define for_each_pwq(pwq, wq)                                                \
        list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,                \
                                 lockdep_is_held(&(wq->mutex)))

#ifdef CONFIG_DEBUG_OBJECTS_WORK

static const struct debug_obj_descr work_debug_descr;

static void *work_debug_hint(void *addr)
{
        return ((struct work_struct *) addr)->func;
}

static bool work_is_static_object(void *addr)
{
        struct work_struct *work = addr;

        return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
}

/*
 * fixup_init is called when:
 * - an active object is initialized
 */
static bool work_fixup_init(void *addr, enum debug_obj_state state)
{
        struct work_struct *work = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                cancel_work_sync(work);
                debug_object_init(work, &work_debug_descr);
                return true;
        default:
                return false;
        }
}

/*
 * fixup_free is called when:
 * - an active object is freed
 */
static bool work_fixup_free(void *addr, enum debug_obj_state state)
{
        struct work_struct *work = addr;

        switch (state) {
        case ODEBUG_STATE_ACTIVE:
                cancel_work_sync(work);
                debug_object_free(work, &work_debug_descr);
                return true;
        default:
                return false;
        }
}

static const struct debug_obj_descr work_debug_descr = {
        .name                = "work_struct",
        .debug_hint        = work_debug_hint,
        .is_static_object = work_is_static_object,
        .fixup_init        = work_fixup_init,
        .fixup_free        = work_fixup_free,
};

static inline void debug_work_activate(struct work_struct *work)
{
        debug_object_activate(work, &work_debug_descr);
}

static inline void debug_work_deactivate(struct work_struct *work)
{
        debug_object_deactivate(work, &work_debug_descr);
}

void __init_work(struct work_struct *work, int onstack)
{
        if (onstack)
                debug_object_init_on_stack(work, &work_debug_descr);
        else
                debug_object_init(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(__init_work);

void destroy_work_on_stack(struct work_struct *work)
{
        debug_object_free(work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_work_on_stack);

void destroy_delayed_work_on_stack(struct delayed_work *work)
{
        timer_destroy_on_stack(&work->timer);
        debug_object_free(&work->work, &work_debug_descr);
}
EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);

#else
static inline void debug_work_activate(struct work_struct *work) { }
static inline void debug_work_deactivate(struct work_struct *work) { }
#endif

/**
 * worker_pool_assign_id - allocate ID and assign it to @pool
 * @pool: the pool pointer of interest
 *
 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
 * successfully, -errno on failure.
 */
static int worker_pool_assign_id(struct worker_pool *pool)
{
        int ret;

        lockdep_assert_held(&wq_pool_mutex);

        ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
                        GFP_KERNEL);
        if (ret >= 0) {
                pool->id = ret;
                return 0;
        }
        return ret;
}

static struct pool_workqueue __rcu **
unbound_pwq_slot(struct workqueue_struct *wq, int cpu)
{
       if (cpu >= 0)
               return per_cpu_ptr(wq->cpu_pwq, cpu);
       else
               return &wq->dfl_pwq;
}

/* @cpu < 0 for dfl_pwq */
static struct pool_workqueue *unbound_pwq(struct workqueue_struct *wq, int cpu)
{
        return rcu_dereference_check(*unbound_pwq_slot(wq, cpu),
                                     lockdep_is_held(&wq_pool_mutex) ||
                                     lockdep_is_held(&wq->mutex));
}

/**
 * unbound_effective_cpumask - effective cpumask of an unbound workqueue
 * @wq: workqueue of interest
 *
 * @wq->unbound_attrs->cpumask contains the cpumask requested by the user which
 * is masked with wq_unbound_cpumask to determine the effective cpumask. The
 * default pwq is always mapped to the pool with the current effective cpumask.
 */
static struct cpumask *unbound_effective_cpumask(struct workqueue_struct *wq)
{
        return unbound_pwq(wq, -1)->pool->attrs->__pod_cpumask;
}

static unsigned int work_color_to_flags(int color)
{
        return color << WORK_STRUCT_COLOR_SHIFT;
}

static int get_work_color(unsigned long work_data)
{
        return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
                ((1 << WORK_STRUCT_COLOR_BITS) - 1);
}

static int work_next_color(int color)
{
        return (color + 1) % WORK_NR_COLORS;
}

static unsigned long pool_offq_flags(struct worker_pool *pool)
{
        return (pool->flags & POOL_BH) ? WORK_OFFQ_BH : 0;
}

/*
 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
 * contain the pointer to the queued pwq.  Once execution starts, the flag
 * is cleared and the high bits contain OFFQ flags and pool ID.
 *
 * set_work_pwq(), set_work_pool_and_clear_pending() and mark_work_canceling()
 * can be used to set the pwq, pool or clear work->data. These functions should
 * only be called while the work is owned - ie. while the PENDING bit is set.
 *
 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
 * corresponding to a work.  Pool is available once the work has been
 * queued anywhere after initialization until it is sync canceled.  pwq is
 * available only while the work item is queued.
 */
static inline void set_work_data(struct work_struct *work, unsigned long data)
{
        WARN_ON_ONCE(!work_pending(work));
        atomic_long_set(&work->data, data | work_static(work));
}

static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
                         unsigned long flags)
{
        set_work_data(work, (unsigned long)pwq | WORK_STRUCT_PENDING |
                      WORK_STRUCT_PWQ | flags);
}

static void set_work_pool_and_keep_pending(struct work_struct *work,
                                           int pool_id, unsigned long flags)
{
        set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
                      WORK_STRUCT_PENDING | flags);
}

static void set_work_pool_and_clear_pending(struct work_struct *work,
                                            int pool_id, unsigned long flags)
{
        /*
         * The following wmb is paired with the implied mb in
         * test_and_set_bit(PENDING) and ensures all updates to @work made
         * here are visible to and precede any updates by the next PENDING
         * owner.
         */
        smp_wmb();
        set_work_data(work, ((unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT) |
                      flags);
        /*
         * The following mb guarantees that previous clear of a PENDING bit
         * will not be reordered with any speculative LOADS or STORES from
         * work->current_func, which is executed afterwards.  This possible
         * reordering can lead to a missed execution on attempt to queue
         * the same @work.  E.g. consider this case:
         *
         *   CPU#0                         CPU#1
         *   ----------------------------  --------------------------------
         *
         * 1  STORE event_indicated
         * 2  queue_work_on() {
         * 3    test_and_set_bit(PENDING)
         * 4 }                             set_..._and_clear_pending() {
         * 5                                 set_work_data() # clear bit
         * 6                                 smp_mb()
         * 7                               work->current_func() {
         * 8                                      LOAD event_indicated
         *                                   }
         *
         * Without an explicit full barrier speculative LOAD on line 8 can
         * be executed before CPU#0 does STORE on line 1.  If that happens,
         * CPU#0 observes the PENDING bit is still set and new execution of
         * a @work is not queued in a hope, that CPU#1 will eventually
         * finish the queued @work.  Meanwhile CPU#1 does not see
         * event_indicated is set, because speculative LOAD was executed
         * before actual STORE.
         */
        smp_mb();
}

static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
{
        return (struct pool_workqueue *)(data & WORK_STRUCT_PWQ_MASK);
}

static struct pool_workqueue *get_work_pwq(struct work_struct *work)
{
        unsigned long data = atomic_long_read(&work->data);

        if (data & WORK_STRUCT_PWQ)
                return work_struct_pwq(data);
        else
                return NULL;
}

/**
 * get_work_pool - return the worker_pool a given work was associated with
 * @work: the work item of interest
 *
 * Pools are created and destroyed under wq_pool_mutex, and allows read
 * access under RCU read lock.  As such, this function should be
 * called under wq_pool_mutex or inside of a rcu_read_lock() region.
 *
 * All fields of the returned pool are accessible as long as the above
 * mentioned locking is in effect.  If the returned pool needs to be used
 * beyond the critical section, the caller is responsible for ensuring the
 * returned pool is and stays online.
 *
 * Return: The worker_pool @work was last associated with.  %NULL if none.
 */
static struct worker_pool *get_work_pool(struct work_struct *work)
{
        unsigned long data = atomic_long_read(&work->data);
        int pool_id;

        assert_rcu_or_pool_mutex();

        if (data & WORK_STRUCT_PWQ)
                return work_struct_pwq(data)->pool;

        pool_id = data >> WORK_OFFQ_POOL_SHIFT;
        if (pool_id == WORK_OFFQ_POOL_NONE)
                return NULL;

        return idr_find(&worker_pool_idr, pool_id);
}

static unsigned long shift_and_mask(unsigned long v, u32 shift, u32 bits)
{
        return (v >> shift) & ((1U << bits) - 1);
}

static void work_offqd_unpack(struct work_offq_data *offqd, unsigned long data)
{
        WARN_ON_ONCE(data & WORK_STRUCT_PWQ);

        offqd->pool_id = shift_and_mask(data, WORK_OFFQ_POOL_SHIFT,
                                        WORK_OFFQ_POOL_BITS);
        offqd->disable = shift_and_mask(data, WORK_OFFQ_DISABLE_SHIFT,
                                        WORK_OFFQ_DISABLE_BITS);
        offqd->flags = data & WORK_OFFQ_FLAG_MASK;
}

static unsigned long work_offqd_pack_flags(struct work_offq_data *offqd)
{
        return ((unsigned long)offqd->disable << WORK_OFFQ_DISABLE_SHIFT) |
                ((unsigned long)offqd->flags);
}

/*
 * Policy functions.  These define the policies on how the global worker
 * pools are managed.  Unless noted otherwise, these functions assume that
 * they're being called with pool->lock held.
 */

/*
 * Need to wake up a worker?  Called from anything but currently
 * running workers.
 *
 * Note that, because unbound workers never contribute to nr_running, this
 * function will always return %true for unbound pools as long as the
 * worklist isn't empty.
 */
static bool need_more_worker(struct worker_pool *pool)
{
        return !list_empty(&pool->worklist) && !pool->nr_running;
}

/* Can I start working?  Called from busy but !running workers. */
static bool may_start_working(struct worker_pool *pool)
{
        return pool->nr_idle;
}

/* Do I need to keep working?  Called from currently running workers. */
static bool keep_working(struct worker_pool *pool)
{
        return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
}

/* Do we need a new worker?  Called from manager. */
static bool need_to_create_worker(struct worker_pool *pool)
{
        return need_more_worker(pool) && !may_start_working(pool);
}

/* Do we have too many workers and should some go away? */
static bool too_many_workers(struct worker_pool *pool)
{
        bool managing = pool->flags & POOL_MANAGER_ACTIVE;
        int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
        int nr_busy = pool->nr_workers - nr_idle;

        return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
}

/**
 * worker_set_flags - set worker flags and adjust nr_running accordingly
 * @worker: self
 * @flags: flags to set
 *
 * Set @flags in @worker->flags and adjust nr_running accordingly.
 */
static inline void worker_set_flags(struct worker *worker, unsigned int flags)
{
        struct worker_pool *pool = worker->pool;

        lockdep_assert_held(&pool->lock);

        /* If transitioning into NOT_RUNNING, adjust nr_running. */
        if ((flags & WORKER_NOT_RUNNING) &&
            !(worker->flags & WORKER_NOT_RUNNING)) {
                pool->nr_running--;
        }

        worker->flags |= flags;
}

/**
 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
 * @worker: self
 * @flags: flags to clear
 *
 * Clear @flags in @worker->flags and adjust nr_running accordingly.
 */
static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
{
        struct worker_pool *pool = worker->pool;
        unsigned int oflags = worker->flags;

        lockdep_assert_held(&pool->lock);

        worker->flags &= ~flags;

        /*
         * If transitioning out of NOT_RUNNING, increment nr_running.  Note
         * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
         * of multiple flags, not a single flag.
         */
        if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
                if (!(worker->flags & WORKER_NOT_RUNNING))
                        pool->nr_running++;
}

/* Return the first idle worker.  Called with pool->lock held. */
static struct worker *first_idle_worker(struct worker_pool *pool)
{
        if (unlikely(list_empty(&pool->idle_list)))
                return NULL;

        return list_first_entry(&pool->idle_list, struct worker, entry);
}

/**
 * worker_enter_idle - enter idle state
 * @worker: worker which is entering idle state
 *
 * @worker is entering idle state.  Update stats and idle timer if
 * necessary.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock).
 */
static void worker_enter_idle(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
            WARN_ON_ONCE(!list_empty(&worker->entry) &&
                         (worker->hentry.next || worker->hentry.pprev)))
                return;

        /* can't use worker_set_flags(), also called from create_worker() */
        worker->flags |= WORKER_IDLE;
        pool->nr_idle++;
        worker->last_active = jiffies;

        /* idle_list is LIFO */
        list_add(&worker->entry, &pool->idle_list);

        if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
                mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);

        /* Sanity check nr_running. */
        WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
}

/**
 * worker_leave_idle - leave idle state
 * @worker: worker which is leaving idle state
 *
 * @worker is leaving idle state.  Update stats.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock).
 */
static void worker_leave_idle(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
                return;
        worker_clr_flags(worker, WORKER_IDLE);
        pool->nr_idle--;
        list_del_init(&worker->entry);
}

/**
 * find_worker_executing_work - find worker which is executing a work
 * @pool: pool of interest
 * @work: work to find worker for
 *
 * Find a worker which is executing @work on @pool by searching
 * @pool->busy_hash which is keyed by the address of @work.  For a worker
 * to match, its current execution should match the address of @work and
 * its work function.  This is to avoid unwanted dependency between
 * unrelated work executions through a work item being recycled while still
 * being executed.
 *
 * This is a bit tricky.  A work item may be freed once its execution
 * starts and nothing prevents the freed area from being recycled for
 * another work item.  If the same work item address ends up being reused
 * before the original execution finishes, workqueue will identify the
 * recycled work item as currently executing and make it wait until the
 * current execution finishes, introducing an unwanted dependency.
 *
 * This function checks the work item address and work function to avoid
 * false positives.  Note that this isn't complete as one may construct a
 * work function which can introduce dependency onto itself through a
 * recycled work item.  Well, if somebody wants to shoot oneself in the
 * foot that badly, there's only so much we can do, and if such deadlock
 * actually occurs, it should be easy to locate the culprit work function.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 *
 * Return:
 * Pointer to worker which is executing @work if found, %NULL
 * otherwise.
 */
static struct worker *find_worker_executing_work(struct worker_pool *pool,
                                                 struct work_struct *work)
{
        struct worker *worker;

        hash_for_each_possible(pool->busy_hash, worker, hentry,
                               (unsigned long)work)
                if (worker->current_work == work &&
                    worker->current_func == work->func)
                        return worker;

        return NULL;
}

/**
 * move_linked_works - move linked works to a list
 * @work: start of series of works to be scheduled
 * @head: target list to append @work to
 * @nextp: out parameter for nested worklist walking
 *
 * Schedule linked works starting from @work to @head. Work series to be
 * scheduled starts at @work and includes any consecutive work with
 * WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on
 * @nextp.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void move_linked_works(struct work_struct *work, struct list_head *head,
                              struct work_struct **nextp)
{
        struct work_struct *n;

        /*
         * Linked worklist will always end before the end of the list,
         * use NULL for list head.
         */
        list_for_each_entry_safe_from(work, n, NULL, entry) {
                list_move_tail(&work->entry, head);
                if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
                        break;
        }

        /*
         * If we're already inside safe list traversal and have moved
         * multiple works to the scheduled queue, the next position
         * needs to be updated.
         */
        if (nextp)
                *nextp = n;
}

/**
 * assign_work - assign a work item and its linked work items to a worker
 * @work: work to assign
 * @worker: worker to assign to
 * @nextp: out parameter for nested worklist walking
 *
 * Assign @work and its linked work items to @worker. If @work is already being
 * executed by another worker in the same pool, it'll be punted there.
 *
 * If @nextp is not NULL, it's updated to point to the next work of the last
 * scheduled work. This allows assign_work() to be nested inside
 * list_for_each_entry_safe().
 *
 * Returns %true if @work was successfully assigned to @worker. %false if @work
 * was punted to another worker already executing it.
 */
static bool assign_work(struct work_struct *work, struct worker *worker,
                        struct work_struct **nextp)
{
        struct worker_pool *pool = worker->pool;
        struct worker *collision;

        lockdep_assert_held(&pool->lock);

        /*
         * A single work shouldn't be executed concurrently by multiple workers.
         * __queue_work() ensures that @work doesn't jump to a different pool
         * while still running in the previous pool. Here, we should ensure that
         * @work is not executed concurrently by multiple workers from the same
         * pool. Check whether anyone is already processing the work. If so,
         * defer the work to the currently executing one.
         */
        collision = find_worker_executing_work(pool, work);
        if (unlikely(collision)) {
                move_linked_works(work, &collision->scheduled, nextp);
                return false;
        }

        move_linked_works(work, &worker->scheduled, nextp);
        return true;
}

static struct irq_work *bh_pool_irq_work(struct worker_pool *pool)
{
        int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0;

        return &per_cpu(bh_pool_irq_works, pool->cpu)[high];
}

static void kick_bh_pool(struct worker_pool *pool)
{
#ifdef CONFIG_SMP
        /* see drain_dead_softirq_workfn() for BH_DRAINING */
        if (unlikely(pool->cpu != smp_processor_id() &&
                     !(pool->flags & POOL_BH_DRAINING))) {
                irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);
                return;
        }
#endif
        if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                raise_softirq_irqoff(HI_SOFTIRQ);
        else
                raise_softirq_irqoff(TASKLET_SOFTIRQ);
}

/**
 * kick_pool - wake up an idle worker if necessary
 * @pool: pool to kick
 *
 * @pool may have pending work items. Wake up worker if necessary. Returns
 * whether a worker was woken up.
 */
static bool kick_pool(struct worker_pool *pool)
{
        struct worker *worker = first_idle_worker(pool);
        struct task_struct *p;

        lockdep_assert_held(&pool->lock);

        if (!need_more_worker(pool) || !worker)
                return false;

        if (pool->flags & POOL_BH) {
                kick_bh_pool(pool);
                return true;
        }

        p = worker->task;

#ifdef CONFIG_SMP
        /*
         * Idle @worker is about to execute @work and waking up provides an
         * opportunity to migrate @worker at a lower cost by setting the task's
         * wake_cpu field. Let's see if we want to move @worker to improve
         * execution locality.
         *
         * We're waking the worker that went idle the latest and there's some
         * chance that @worker is marked idle but hasn't gone off CPU yet. If
         * so, setting the wake_cpu won't do anything. As this is a best-effort
         * optimization and the race window is narrow, let's leave as-is for
         * now. If this becomes pronounced, we can skip over workers which are
         * still on cpu when picking an idle worker.
         *
         * If @pool has non-strict affinity, @worker might have ended up outside
         * its affinity scope. Repatriate.
         */
        if (!pool->attrs->affn_strict &&
            !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {
                struct work_struct *work = list_first_entry(&pool->worklist,
                                                struct work_struct, entry);
                int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,
                                                          cpu_online_mask);
                if (wake_cpu < nr_cpu_ids) {
                        p->wake_cpu = wake_cpu;
                        get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;
                }
        }
#endif
        wake_up_process(p);
        return true;
}

#ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT

/*
 * Concurrency-managed per-cpu work items that hog CPU for longer than
 * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism,
 * which prevents them from stalling other concurrency-managed work items. If a
 * work function keeps triggering this mechanism, it's likely that the work item
 * should be using an unbound workqueue instead.
 *
 * wq_cpu_intensive_report() tracks work functions which trigger such conditions
 * and report them so that they can be examined and converted to use unbound
 * workqueues as appropriate. To avoid flooding the console, each violating work
 * function is tracked and reported with exponential backoff.
 */
#define WCI_MAX_ENTS 128

struct wci_ent {
        work_func_t                func;
        atomic64_t                cnt;
        struct hlist_node        hash_node;
};

static struct wci_ent wci_ents[WCI_MAX_ENTS];
static int wci_nr_ents;
static DEFINE_RAW_SPINLOCK(wci_lock);
static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS));

static struct wci_ent *wci_find_ent(work_func_t func)
{
        struct wci_ent *ent;

        hash_for_each_possible_rcu(wci_hash, ent, hash_node,
                                   (unsigned long)func) {
                if (ent->func == func)
                        return ent;
        }
        return NULL;
}

static void wq_cpu_intensive_report(work_func_t func)
{
        struct wci_ent *ent;

restart:
        ent = wci_find_ent(func);
        if (ent) {
                u64 cnt;

                /*
                 * Start reporting from the warning_thresh and back off
                 * exponentially.
                 */
                cnt = atomic64_inc_return_relaxed(&ent->cnt);
                if (wq_cpu_intensive_warning_thresh &&
                    cnt >= wq_cpu_intensive_warning_thresh &&
                    is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))
                        printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",
                                        ent->func, wq_cpu_intensive_thresh_us,
                                        atomic64_read(&ent->cnt));
                return;
        }

        /*
         * @func is a new violation. Allocate a new entry for it. If wcn_ents[]
         * is exhausted, something went really wrong and we probably made enough
         * noise already.
         */
        if (wci_nr_ents >= WCI_MAX_ENTS)
                return;

        raw_spin_lock(&wci_lock);

        if (wci_nr_ents >= WCI_MAX_ENTS) {
                raw_spin_unlock(&wci_lock);
                return;
        }

        if (wci_find_ent(func)) {
                raw_spin_unlock(&wci_lock);
                goto restart;
        }

        ent = &wci_ents[wci_nr_ents++];
        ent->func = func;
        atomic64_set(&ent->cnt, 0);
        hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func);

        raw_spin_unlock(&wci_lock);

        goto restart;
}

#else        /* CONFIG_WQ_CPU_INTENSIVE_REPORT */
static void wq_cpu_intensive_report(work_func_t func) {}
#endif        /* CONFIG_WQ_CPU_INTENSIVE_REPORT */

/**
 * wq_worker_running - a worker is running again
 * @task: task waking up
 *
 * This function is called when a worker returns from schedule()
 */
void wq_worker_running(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);

        if (!READ_ONCE(worker->sleeping))
                return;

        /*
         * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
         * and the nr_running increment below, we may ruin the nr_running reset
         * and leave with an unexpected pool->nr_running == 1 on the newly unbound
         * pool. Protect against such race.
         */
        preempt_disable();
        if (!(worker->flags & WORKER_NOT_RUNNING))
                worker->pool->nr_running++;
        preempt_enable();

        /*
         * CPU intensive auto-detection cares about how long a work item hogged
         * CPU without sleeping. Reset the starting timestamp on wakeup.
         */
        worker->current_at = worker->task->se.sum_exec_runtime;

        WRITE_ONCE(worker->sleeping, 0);
}

/**
 * wq_worker_sleeping - a worker is going to sleep
 * @task: task going to sleep
 *
 * This function is called from schedule() when a busy worker is
 * going to sleep.
 */
void wq_worker_sleeping(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);
        struct worker_pool *pool;

        /*
         * Rescuers, which may not have all the fields set up like normal
         * workers, also reach here, let's not access anything before
         * checking NOT_RUNNING.
         */
        if (worker->flags & WORKER_NOT_RUNNING)
                return;

        pool = worker->pool;

        /* Return if preempted before wq_worker_running() was reached */
        if (READ_ONCE(worker->sleeping))
                return;

        WRITE_ONCE(worker->sleeping, 1);
        raw_spin_lock_irq(&pool->lock);

        /*
         * Recheck in case unbind_workers() preempted us. We don't
         * want to decrement nr_running after the worker is unbound
         * and nr_running has been reset.
         */
        if (worker->flags & WORKER_NOT_RUNNING) {
                raw_spin_unlock_irq(&pool->lock);
                return;
        }

        pool->nr_running--;
        if (kick_pool(pool))
                worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++;

        raw_spin_unlock_irq(&pool->lock);
}

/**
 * wq_worker_tick - a scheduler tick occurred while a kworker is running
 * @task: task currently running
 *
 * Called from sched_tick(). We're in the IRQ context and the current
 * worker's fields which follow the 'K' locking rule can be accessed safely.
 */
void wq_worker_tick(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);
        struct pool_workqueue *pwq = worker->current_pwq;
        struct worker_pool *pool = worker->pool;

        if (!pwq)
                return;

        pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC;

        if (!wq_cpu_intensive_thresh_us)
                return;

        /*
         * If the current worker is concurrency managed and hogged the CPU for
         * longer than wq_cpu_intensive_thresh_us, it's automatically marked
         * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.
         *
         * Set @worker->sleeping means that @worker is in the process of
         * switching out voluntarily and won't be contributing to
         * @pool->nr_running until it wakes up. As wq_worker_sleeping() also
         * decrements ->nr_running, setting CPU_INTENSIVE here can lead to
         * double decrements. The task is releasing the CPU anyway. Let's skip.
         * We probably want to make this prettier in the future.
         */
        if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
            worker->task->se.sum_exec_runtime - worker->current_at <
            wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
                return;

        raw_spin_lock(&pool->lock);

        worker_set_flags(worker, WORKER_CPU_INTENSIVE);
        wq_cpu_intensive_report(worker->current_func);
        pwq->stats[PWQ_STAT_CPU_INTENSIVE]++;

        if (kick_pool(pool))
                pwq->stats[PWQ_STAT_CM_WAKEUP]++;

        raw_spin_unlock(&pool->lock);
}

/**
 * wq_worker_last_func - retrieve worker's last work function
 * @task: Task to retrieve last work function of.
 *
 * Determine the last function a worker executed. This is called from
 * the scheduler to get a worker's last known identity.
 *
 * CONTEXT:
 * raw_spin_lock_irq(rq->lock)
 *
 * This function is called during schedule() when a kworker is going
 * to sleep. It's used by psi to identify aggregation workers during
 * dequeuing, to allow periodic aggregation to shut-off when that
 * worker is the last task in the system or cgroup to go to sleep.
 *
 * As this function doesn't involve any workqueue-related locking, it
 * only returns stable values when called from inside the scheduler's
 * queuing and dequeuing paths, when @task, which must be a kworker,
 * is guaranteed to not be processing any works.
 *
 * Return:
 * The last work function %current executed as a worker, NULL if it
 * hasn't executed any work yet.
 */
work_func_t wq_worker_last_func(struct task_struct *task)
{
        struct worker *worker = kthread_data(task);

        return worker->last_func;
}

/**
 * wq_node_nr_active - Determine wq_node_nr_active to use
 * @wq: workqueue of interest
 * @node: NUMA node, can be %NUMA_NO_NODE
 *
 * Determine wq_node_nr_active to use for @wq on @node. Returns:
 *
 * - %NULL for per-cpu workqueues as they don't need to use shared nr_active.
 *
 * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE.
 *
 * - Otherwise, node_nr_active[@node].
 */
static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,
                                                   int node)
{
        if (!(wq->flags & WQ_UNBOUND))
                return NULL;

        if (node == NUMA_NO_NODE)
                node = nr_node_ids;

        return wq->node_nr_active[node];
}

/**
 * wq_update_node_max_active - Update per-node max_actives to use
 * @wq: workqueue to update
 * @off_cpu: CPU that's going down, -1 if a CPU is not going down
 *
 * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is
 * distributed among nodes according to the proportions of numbers of online
 * cpus. The result is always between @wq->min_active and max_active.
 */
static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu)
{
        struct cpumask *effective = unbound_effective_cpumask(wq);
        int min_active = READ_ONCE(wq->min_active);
        int max_active = READ_ONCE(wq->max_active);
        int total_cpus, node;

        lockdep_assert_held(&wq->mutex);

        if (!wq_topo_initialized)
                return;

        if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))
                off_cpu = -1;

        total_cpus = cpumask_weight_and(effective, cpu_online_mask);
        if (off_cpu >= 0)
                total_cpus--;

        /* If all CPUs of the wq get offline, use the default values */
        if (unlikely(!total_cpus)) {
                for_each_node(node)
                        wq_node_nr_active(wq, node)->max = min_active;

                wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
                return;
        }

        for_each_node(node) {
                int node_cpus;

                node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));
                if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)
                        node_cpus--;

                wq_node_nr_active(wq, node)->max =
                        clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),
                              min_active, max_active);
        }

        wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;
}

/**
 * get_pwq - get an extra reference on the specified pool_workqueue
 * @pwq: pool_workqueue to get
 *
 * Obtain an extra reference on @pwq.  The caller should guarantee that
 * @pwq has positive refcnt and be holding the matching pool->lock.
 */
static void get_pwq(struct pool_workqueue *pwq)
{
        lockdep_assert_held(&pwq->pool->lock);
        WARN_ON_ONCE(pwq->refcnt <= 0);
        pwq->refcnt++;
}

/**
 * put_pwq - put a pool_workqueue reference
 * @pwq: pool_workqueue to put
 *
 * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
 * destruction.  The caller should be holding the matching pool->lock.
 */
static void put_pwq(struct pool_workqueue *pwq)
{
        lockdep_assert_held(&pwq->pool->lock);
        if (likely(--pwq->refcnt))
                return;
        /*
         * @pwq can't be released under pool->lock, bounce to a dedicated
         * kthread_worker to avoid A-A deadlocks.
         */
        kthread_queue_work(pwq_release_worker, &pwq->release_work);
}

/**
 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
 * @pwq: pool_workqueue to put (can be %NULL)
 *
 * put_pwq() with locking.  This function also allows %NULL @pwq.
 */
static void put_pwq_unlocked(struct pool_workqueue *pwq)
{
        if (pwq) {
                /*
                 * As both pwqs and pools are RCU protected, the
                 * following lock operations are safe.
                 */
                raw_spin_lock_irq(&pwq->pool->lock);
                put_pwq(pwq);
                raw_spin_unlock_irq(&pwq->pool->lock);
        }
}

static bool pwq_is_empty(struct pool_workqueue *pwq)
{
        return !pwq->nr_active && list_empty(&pwq->inactive_works);
}

static void __pwq_activate_work(struct pool_workqueue *pwq,
                                struct work_struct *work)
{
        unsigned long *wdb = work_data_bits(work);

        WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));
        trace_workqueue_activate_work(work);
        if (list_empty(&pwq->pool->worklist))
                pwq->pool->watchdog_ts = jiffies;
        move_linked_works(work, &pwq->pool->worklist, NULL);
        __clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);
}

static bool tryinc_node_nr_active(struct wq_node_nr_active *nna)
{
        int max = READ_ONCE(nna->max);

        while (true) {
                int old, tmp;

                old = atomic_read(&nna->nr);
                if (old >= max)
                        return false;
                tmp = atomic_cmpxchg_relaxed(&nna->nr, old, old + 1);
                if (tmp == old)
                        return true;
        }
}

/**
 * pwq_tryinc_nr_active - Try to increment nr_active for a pwq
 * @pwq: pool_workqueue of interest
 * @fill: max_active may have increased, try to increase concurrency level
 *
 * Try to increment nr_active for @pwq. Returns %true if an nr_active count is
 * successfully obtained. %false otherwise.
 */
static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill)
{
        struct workqueue_struct *wq = pwq->wq;
        struct worker_pool *pool = pwq->pool;
        struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);
        bool obtained = false;

        lockdep_assert_held(&pool->lock);

        if (!nna) {
                /* BH or per-cpu workqueue, pwq->nr_active is sufficient */
                obtained = pwq->nr_active < READ_ONCE(wq->max_active);
                goto out;
        }

        if (unlikely(pwq->plugged))
                return false;

        /*
         * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is
         * already waiting on $nna, pwq_dec_nr_active() will maintain the
         * concurrency level. Don't jump the line.
         *
         * We need to ignore the pending test after max_active has increased as
         * pwq_dec_nr_active() can only maintain the concurrency level but not
         * increase it. This is indicated by @fill.
         */
        if (!list_empty(&pwq->pending_node) && likely(!fill))
                goto out;

        obtained = tryinc_node_nr_active(nna);
        if (obtained)
                goto out;

        /*
         * Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs
         * and try again. The smp_mb() is paired with the implied memory barrier
         * of atomic_dec_return() in pwq_dec_nr_active() to ensure that either
         * we see the decremented $nna->nr or they see non-empty
         * $nna->pending_pwqs.
         */
        raw_spin_lock(&nna->lock);

        if (list_empty(&pwq->pending_node))
                list_add_tail(&pwq->pending_node, &nna->pending_pwqs);
        else if (likely(!fill))
                goto out_unlock;

        smp_mb();

        obtained = tryinc_node_nr_active(nna);

        /*
         * If @fill, @pwq might have already been pending. Being spuriously
         * pending in cold paths doesn't affect anything. Let's leave it be.
         */
        if (obtained && likely(!fill))
                list_del_init(&pwq->pending_node);

out_unlock:
        raw_spin_unlock(&nna->lock);
out:
        if (obtained)
                pwq->nr_active++;
        return obtained;
}

/**
 * pwq_activate_first_inactive - Activate the first inactive work item on a pwq
 * @pwq: pool_workqueue of interest
 * @fill: max_active may have increased, try to increase concurrency level
 *
 * Activate the first inactive work item of @pwq if available and allowed by
 * max_active limit.
 *
 * Returns %true if an inactive work item has been activated. %false if no
 * inactive work item is found or max_active limit is reached.
 */
static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill)
{
        struct work_struct *work =
                list_first_entry_or_null(&pwq->inactive_works,
                                         struct work_struct, entry);

        if (work && pwq_tryinc_nr_active(pwq, fill)) {
                __pwq_activate_work(pwq, work);
                return true;
        } else {
                return false;
        }
}

/**
 * unplug_oldest_pwq - unplug the oldest pool_workqueue
 * @wq: workqueue_struct where its oldest pwq is to be unplugged
 *
 * This function should only be called for ordered workqueues where only the
 * oldest pwq is unplugged, the others are plugged to suspend execution to
 * ensure proper work item ordering::
 *
 *    dfl_pwq --------------+     [P] - plugged
 *                          |
 *                          v
 *    pwqs -> A -> B [P] -> C [P] (newest)
 *            |    |        |
 *            1    3        5
 *            |    |        |
 *            2    4        6
 *
 * When the oldest pwq is drained and removed, this function should be called
 * to unplug the next oldest one to start its work item execution. Note that
 * pwq's are linked into wq->pwqs with the oldest first, so the first one in
 * the list is the oldest.
 */
static void unplug_oldest_pwq(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;

        lockdep_assert_held(&wq->mutex);

        /* Caller should make sure that pwqs isn't empty before calling */
        pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,
                                       pwqs_node);
        raw_spin_lock_irq(&pwq->pool->lock);
        if (pwq->plugged) {
                pwq->plugged = false;
                if (pwq_activate_first_inactive(pwq, true))
                        kick_pool(pwq->pool);
        }
        raw_spin_unlock_irq(&pwq->pool->lock);
}

/**
 * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active
 * @nna: wq_node_nr_active to activate a pending pwq for
 * @caller_pool: worker_pool the caller is locking
 *
 * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked.
 * @caller_pool may be unlocked and relocked to lock other worker_pools.
 */
static void node_activate_pending_pwq(struct wq_node_nr_active *nna,
                                      struct worker_pool *caller_pool)
{
        struct worker_pool *locked_pool = caller_pool;
        struct pool_workqueue *pwq;
        struct work_struct *work;

        lockdep_assert_held(&caller_pool->lock);

        raw_spin_lock(&nna->lock);
retry:
        pwq = list_first_entry_or_null(&nna->pending_pwqs,
                                       struct pool_workqueue, pending_node);
        if (!pwq)
                goto out_unlock;

        /*
         * If @pwq is for a different pool than @locked_pool, we need to lock
         * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock
         * / lock dance. For that, we also need to release @nna->lock as it's
         * nested inside pool locks.
         */
        if (pwq->pool != locked_pool) {
                raw_spin_unlock(&locked_pool->lock);
                locked_pool = pwq->pool;
                if (!raw_spin_trylock(&locked_pool->lock)) {
                        raw_spin_unlock(&nna->lock);
                        raw_spin_lock(&locked_pool->lock);
                        raw_spin_lock(&nna->lock);
                        goto retry;
                }
        }

        /*
         * $pwq may not have any inactive work items due to e.g. cancellations.
         * Drop it from pending_pwqs and see if there's another one.
         */
        work = list_first_entry_or_null(&pwq->inactive_works,
                                        struct work_struct, entry);
        if (!work) {
                list_del_init(&pwq->pending_node);
                goto retry;
        }

        /*
         * Acquire an nr_active count and activate the inactive work item. If
         * $pwq still has inactive work items, rotate it to the end of the
         * pending_pwqs so that we round-robin through them. This means that
         * inactive work items are not activated in queueing order which is fine
         * given that there has never been any ordering across different pwqs.
         */
        if (likely(tryinc_node_nr_active(nna))) {
                pwq->nr_active++;
                __pwq_activate_work(pwq, work);

                if (list_empty(&pwq->inactive_works))
                        list_del_init(&pwq->pending_node);
                else
                        list_move_tail(&pwq->pending_node, &nna->pending_pwqs);

                /* if activating a foreign pool, make sure it's running */
                if (pwq->pool != caller_pool)
                        kick_pool(pwq->pool);
        }

out_unlock:
        raw_spin_unlock(&nna->lock);
        if (locked_pool != caller_pool) {
                raw_spin_unlock(&locked_pool->lock);
                raw_spin_lock(&caller_pool->lock);
        }
}

/**
 * pwq_dec_nr_active - Retire an active count
 * @pwq: pool_workqueue of interest
 *
 * Decrement @pwq's nr_active and try to activate the first inactive work item.
 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock.
 */
static void pwq_dec_nr_active(struct pool_workqueue *pwq)
{
        struct worker_pool *pool = pwq->pool;
        struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node);

        lockdep_assert_held(&pool->lock);

        /*
         * @pwq->nr_active should be decremented for both percpu and unbound
         * workqueues.
         */
        pwq->nr_active--;

        /*
         * For a percpu workqueue, it's simple. Just need to kick the first
         * inactive work item on @pwq itself.
         */
        if (!nna) {
                pwq_activate_first_inactive(pwq, false);
                return;
        }

        /*
         * If @pwq is for an unbound workqueue, it's more complicated because
         * multiple pwqs and pools may be sharing the nr_active count. When a
         * pwq needs to wait for an nr_active count, it puts itself on
         * $nna->pending_pwqs. The following atomic_dec_return()'s implied
         * memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to
         * guarantee that either we see non-empty pending_pwqs or they see
         * decremented $nna->nr.
         *
         * $nna->max may change as CPUs come online/offline and @pwq->wq's
         * max_active gets updated. However, it is guaranteed to be equal to or
         * larger than @pwq->wq->min_active which is above zero unless freezing.
         * This maintains the forward progress guarantee.
         */
        if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))
                return;

        if (!list_empty(&nna->pending_pwqs))
                node_activate_pending_pwq(nna, pool);
}

/**
 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
 * @pwq: pwq of interest
 * @work_data: work_data of work which left the queue
 *
 * A work either has completed or is removed from pending queue,
 * decrement nr_in_flight of its pwq and handle workqueue flushing.
 *
 * NOTE:
 * For unbound workqueues, this function may temporarily drop @pwq->pool->lock
 * and thus should be called after all other state updates for the in-flight
 * work item is complete.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
{
        int color = get_work_color(work_data);

        if (!(work_data & WORK_STRUCT_INACTIVE))
                pwq_dec_nr_active(pwq);

        pwq->nr_in_flight[color]--;

        /* is flush in progress and are we at the flushing tip? */
        if (likely(pwq->flush_color != color))
                goto out_put;

        /* are there still in-flight works? */
        if (pwq->nr_in_flight[color])
                goto out_put;

        /* this pwq is done, clear flush_color */
        pwq->flush_color = -1;

        /*
         * If this was the last pwq, wake up the first flusher.  It
         * will handle the rest.
         */
        if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
                complete(&pwq->wq->first_flusher->done);
out_put:
        put_pwq(pwq);
}

/**
 * try_to_grab_pending - steal work item from worklist and disable irq
 * @work: work item to steal
 * @cflags: %WORK_CANCEL_ flags
 * @irq_flags: place to store irq state
 *
 * Try to grab PENDING bit of @work.  This function can handle @work in any
 * stable state - idle, on timer or on worklist.
 *
 * Return:
 *
 *  ========        ================================================================
 *  1                if @work was pending and we successfully stole PENDING
 *  0                if @work was idle and we claimed PENDING
 *  -EAGAIN        if PENDING couldn't be grabbed at the moment, safe to busy-retry
 *  ========        ================================================================
 *
 * Note:
 * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
 * interrupted while holding PENDING and @work off queue, irq must be
 * disabled on entry.  This, combined with delayed_work->timer being
 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
 *
 * On successful return, >= 0, irq is disabled and the caller is
 * responsible for releasing it using local_irq_restore(*@irq_flags).
 *
 * This function is safe to call from any context including IRQ handler.
 */
static int try_to_grab_pending(struct work_struct *work, u32 cflags,
                               unsigned long *irq_flags)
{
        struct worker_pool *pool;
        struct pool_workqueue *pwq;

        local_irq_save(*irq_flags);

        /* try to steal the timer if it exists */
        if (cflags & WORK_CANCEL_DELAYED) {
                struct delayed_work *dwork = to_delayed_work(work);

                /*
                 * dwork->timer is irqsafe.  If timer_delete() fails, it's
                 * guaranteed that the timer is not queued anywhere and not
                 * running on the local CPU.
                 */
                if (likely(timer_delete(&dwork->timer)))
                        return 1;
        }

        /* try to claim PENDING the normal way */
        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
                return 0;

        rcu_read_lock();
        /*
         * The queueing is in progress, or it is already queued. Try to
         * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
         */
        pool = get_work_pool(work);
        if (!pool)
                goto fail;

        raw_spin_lock(&pool->lock);
        /*
         * work->data is guaranteed to point to pwq only while the work
         * item is queued on pwq->wq, and both updating work->data to point
         * to pwq on queueing and to pool on dequeueing are done under
         * pwq->pool->lock.  This in turn guarantees that, if work->data
         * points to pwq which is associated with a locked pool, the work
         * item is currently queued on that pool.
         */
        pwq = get_work_pwq(work);
        if (pwq && pwq->pool == pool) {
                unsigned long work_data = *work_data_bits(work);

                debug_work_deactivate(work);

                /*
                 * A cancelable inactive work item must be in the
                 * pwq->inactive_works since a queued barrier can't be
                 * canceled (see the comments in insert_wq_barrier()).
                 *
                 * An inactive work item cannot be deleted directly because
                 * it might have linked barrier work items which, if left
                 * on the inactive_works list, will confuse pwq->nr_active
                 * management later on and cause stall.  Move the linked
                 * barrier work items to the worklist when deleting the grabbed
                 * item. Also keep WORK_STRUCT_INACTIVE in work_data, so that
                 * it doesn't participate in nr_active management in later
                 * pwq_dec_nr_in_flight().
                 */
                if (work_data & WORK_STRUCT_INACTIVE)
                        move_linked_works(work, &pwq->pool->worklist, NULL);

                list_del_init(&work->entry);

                /*
                 * work->data points to pwq iff queued. Let's point to pool. As
                 * this destroys work->data needed by the next step, stash it.
                 */
                set_work_pool_and_keep_pending(work, pool->id,
                                               pool_offq_flags(pool));

                /* must be the last step, see the function comment */
                pwq_dec_nr_in_flight(pwq, work_data);

                raw_spin_unlock(&pool->lock);
                rcu_read_unlock();
                return 1;
        }
        raw_spin_unlock(&pool->lock);
fail:
        rcu_read_unlock();
        local_irq_restore(*irq_flags);
        return -EAGAIN;
}

/**
 * work_grab_pending - steal work item from worklist and disable irq
 * @work: work item to steal
 * @cflags: %WORK_CANCEL_ flags
 * @irq_flags: place to store IRQ state
 *
 * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer
 * or on worklist.
 *
 * Can be called from any context. IRQ is disabled on return with IRQ state
 * stored in *@irq_flags. The caller is responsible for re-enabling it using
 * local_irq_restore().
 *
 * Returns %true if @work was pending. %false if idle.
 */
static bool work_grab_pending(struct work_struct *work, u32 cflags,
                              unsigned long *irq_flags)
{
        int ret;

        while (true) {
                ret = try_to_grab_pending(work, cflags, irq_flags);
                if (ret >= 0)
                        return ret;
                cpu_relax();
        }
}

/**
 * insert_work - insert a work into a pool
 * @pwq: pwq @work belongs to
 * @work: work to insert
 * @head: insertion point
 * @extra_flags: extra WORK_STRUCT_* flags to set
 *
 * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
 * work_struct flags.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
                        struct list_head *head, unsigned int extra_flags)
{
        debug_work_activate(work);

        /* record the work call stack in order to print it in KASAN reports */
        kasan_record_aux_stack(work);

        /* we own @work, set data and link */
        set_work_pwq(work, pwq, extra_flags);
        list_add_tail(&work->entry, head);
        get_pwq(pwq);
}

/*
 * Test whether @work is being queued from another work executing on the
 * same workqueue.
 */
static bool is_chained_work(struct workqueue_struct *wq)
{
        struct worker *worker;

        worker = current_wq_worker();
        /*
         * Return %true iff I'm a worker executing a work item on @wq.  If
         * I'm @worker, it's safe to dereference it without locking.
         */
        return worker && worker->current_pwq->wq == wq;
}

/*
 * When queueing an unbound work item to a wq, prefer local CPU if allowed
 * by wq_unbound_cpumask.  Otherwise, round robin among the allowed ones to
 * avoid perturbing sensitive tasks.
 */
static int wq_select_unbound_cpu(int cpu)
{
        int new_cpu;

        if (likely(!wq_debug_force_rr_cpu)) {
                if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
                        return cpu;
        } else {
                pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");
        }

        new_cpu = __this_cpu_read(wq_rr_cpu_last);
        new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
        if (unlikely(new_cpu >= nr_cpu_ids)) {
                new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
                if (unlikely(new_cpu >= nr_cpu_ids))
                        return cpu;
        }
        __this_cpu_write(wq_rr_cpu_last, new_cpu);

        return new_cpu;
}

static void __queue_work(int cpu, struct workqueue_struct *wq,
                         struct work_struct *work)
{
        struct pool_workqueue *pwq;
        struct worker_pool *last_pool, *pool;
        unsigned int work_flags;
        unsigned int req_cpu = cpu;

        /*
         * While a work item is PENDING && off queue, a task trying to
         * steal the PENDING will busy-loop waiting for it to either get
         * queued or lose PENDING.  Grabbing PENDING and queueing should
         * happen with IRQ disabled.
         */
        lockdep_assert_irqs_disabled();

        /*
         * For a draining wq, only works from the same workqueue are
         * allowed. The __WQ_DESTROYING helps to spot the issue that
         * queues a new work item to a wq after destroy_workqueue(wq).
         */
        if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&
                     WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",
                               work->func, wq->name))) {
                return;
        }
        rcu_read_lock();
retry:
        /* pwq which will be used unless @work is executing elsewhere */
        if (req_cpu == WORK_CPU_UNBOUND) {
                if (wq->flags & WQ_UNBOUND)
                        cpu = wq_select_unbound_cpu(raw_smp_processor_id());
                else
                        cpu = raw_smp_processor_id();
        }

        pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));
        pool = pwq->pool;

        /*
         * If @work was previously on a different pool, it might still be
         * running there, in which case the work needs to be queued on that
         * pool to guarantee non-reentrancy.
         *
         * For ordered workqueue, work items must be queued on the newest pwq
         * for accurate order management.  Guaranteed order also guarantees
         * non-reentrancy.  See the comments above unplug_oldest_pwq().
         */
        last_pool = get_work_pool(work);
        if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {
                struct worker *worker;

                raw_spin_lock(&last_pool->lock);

                worker = find_worker_executing_work(last_pool, work);

                if (worker && worker->current_pwq->wq == wq) {
                        pwq = worker->current_pwq;
                        pool = pwq->pool;
                        WARN_ON_ONCE(pool != last_pool);
                } else {
                        /* meh... not running there, queue here */
                        raw_spin_unlock(&last_pool->lock);
                        raw_spin_lock(&pool->lock);
                }
        } else {
                raw_spin_lock(&pool->lock);
        }

        /*
         * pwq is determined and locked. For unbound pools, we could have raced
         * with pwq release and it could already be dead. If its refcnt is zero,
         * repeat pwq selection. Note that unbound pwqs never die without
         * another pwq replacing it in cpu_pwq or while work items are executing
         * on it, so the retrying is guaranteed to make forward-progress.
         */
        if (unlikely(!pwq->refcnt)) {
                if (wq->flags & WQ_UNBOUND) {
                        raw_spin_unlock(&pool->lock);
                        cpu_relax();
                        goto retry;
                }
                /* oops */
                WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
                          wq->name, cpu);
        }

        /* pwq determined, queue */
        trace_workqueue_queue_work(req_cpu, pwq, work);

        if (WARN_ON(!list_empty(&work->entry)))
                goto out;

        pwq->nr_in_flight[pwq->work_color]++;
        work_flags = work_color_to_flags(pwq->work_color);

        /*
         * Limit the number of concurrently active work items to max_active.
         * @work must also queue behind existing inactive work items to maintain
         * ordering when max_active changes. See wq_adjust_max_active().
         */
        if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {
                if (list_empty(&pool->worklist))
                        pool->watchdog_ts = jiffies;

                trace_workqueue_activate_work(work);
                insert_work(pwq, work, &pool->worklist, work_flags);
                kick_pool(pool);
        } else {
                work_flags |= WORK_STRUCT_INACTIVE;
                insert_work(pwq, work, &pwq->inactive_works, work_flags);
        }

out:
        raw_spin_unlock(&pool->lock);
        rcu_read_unlock();
}

static bool clear_pending_if_disabled(struct work_struct *work)
{
        unsigned long data = *work_data_bits(work);
        struct work_offq_data offqd;

        if (likely((data & WORK_STRUCT_PWQ) ||
                   !(data & WORK_OFFQ_DISABLE_MASK)))
                return false;

        work_offqd_unpack(&offqd, data);
        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        return true;
}

/**
 * queue_work_on - queue work on specific cpu
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @work: work to queue
 *
 * We queue the work to a specific CPU, the caller must ensure it
 * can't go away.  Callers that fail to ensure that the specified
 * CPU cannot go away will execute on a randomly chosen CPU.
 * But note well that callers specifying a CPU that never has been
 * online will get a splat.
 *
 * Return: %false if @work was already on a queue, %true otherwise.
 */
bool queue_work_on(int cpu, struct workqueue_struct *wq,
                   struct work_struct *work)
{
        bool ret = false;
        unsigned long irq_flags;

        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                __queue_work(cpu, wq, work);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL(queue_work_on);

/**
 * select_numa_node_cpu - Select a CPU based on NUMA node
 * @node: NUMA node ID that we want to select a CPU from
 *
 * This function will attempt to find a "random" cpu available on a given
 * node. If there are no CPUs available on the given node it will return
 * WORK_CPU_UNBOUND indicating that we should just schedule to any
 * available CPU if we need to schedule this work.
 */
static int select_numa_node_cpu(int node)
{
        int cpu;

        /* Delay binding to CPU if node is not valid or online */
        if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
                return WORK_CPU_UNBOUND;

        /* Use local node/cpu if we are already there */
        cpu = raw_smp_processor_id();
        if (node == cpu_to_node(cpu))
                return cpu;

        /* Use "random" otherwise know as "first" online CPU of node */
        cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);

        /* If CPU is valid return that, otherwise just defer */
        return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
}

/**
 * queue_work_node - queue work on a "random" cpu for a given NUMA node
 * @node: NUMA node that we are targeting the work for
 * @wq: workqueue to use
 * @work: work to queue
 *
 * We queue the work to a "random" CPU within a given NUMA node. The basic
 * idea here is to provide a way to somehow associate work with a given
 * NUMA node.
 *
 * This function will only make a best effort attempt at getting this onto
 * the right NUMA node. If no node is requested or the requested node is
 * offline then we just fall back to standard queue_work behavior.
 *
 * Currently the "random" CPU ends up being the first available CPU in the
 * intersection of cpu_online_mask and the cpumask of the node, unless we
 * are running on the node. In that case we just use the current CPU.
 *
 * Return: %false if @work was already on a queue, %true otherwise.
 */
bool queue_work_node(int node, struct workqueue_struct *wq,
                     struct work_struct *work)
{
        unsigned long irq_flags;
        bool ret = false;

        /*
         * This current implementation is specific to unbound workqueues.
         * Specifically we only return the first available CPU for a given
         * node instead of cycling through individual CPUs within the node.
         *
         * If this is used with a per-cpu workqueue then the logic in
         * workqueue_select_cpu_near would need to be updated to allow for
         * some round robin type logic.
         */
        WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));

        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                int cpu = select_numa_node_cpu(node);

                __queue_work(cpu, wq, work);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL_GPL(queue_work_node);

void delayed_work_timer_fn(struct timer_list *t)
{
        struct delayed_work *dwork = timer_container_of(dwork, t, timer);

        /* should have been called from irqsafe timer with irq already off */
        __queue_work(dwork->cpu, dwork->wq, &dwork->work);
}
EXPORT_SYMBOL(delayed_work_timer_fn);

static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
                                struct delayed_work *dwork, unsigned long delay)
{
        struct timer_list *timer = &dwork->timer;
        struct work_struct *work = &dwork->work;

        WARN_ON_ONCE(!wq);
        WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
        WARN_ON_ONCE(timer_pending(timer));
        WARN_ON_ONCE(!list_empty(&work->entry));

        /*
         * If @delay is 0, queue @dwork->work immediately.  This is for
         * both optimization and correctness.  The earliest @timer can
         * expire is on the closest next tick and delayed_work users depend
         * on that there's no such delay when @delay is 0.
         */
        if (!delay) {
                __queue_work(cpu, wq, &dwork->work);
                return;
        }

        WARN_ON_ONCE(cpu != WORK_CPU_UNBOUND && !cpu_online(cpu));
        dwork->wq = wq;
        dwork->cpu = cpu;
        timer->expires = jiffies + delay;

        if (housekeeping_enabled(HK_TYPE_TIMER)) {
                /* If the current cpu is a housekeeping cpu, use it. */
                cpu = smp_processor_id();
                if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))
                        cpu = housekeeping_any_cpu(HK_TYPE_TIMER);
                add_timer_on(timer, cpu);
        } else {
                if (likely(cpu == WORK_CPU_UNBOUND))
                        add_timer_global(timer);
                else
                        add_timer_on(timer, cpu);
        }
}

/**
 * queue_delayed_work_on - queue work on specific CPU after delay
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @dwork: work to queue
 * @delay: number of jiffies to wait before queueing
 *
 * We queue the delayed_work to a specific CPU, for non-zero delays the
 * caller must ensure it is online and can't go away. Callers that fail
 * to ensure this, may get @dwork->timer queued to an offlined CPU and
 * this will prevent queueing of @dwork->work unless the offlined CPU
 * becomes online again.
 *
 * Return: %false if @work was already on a queue, %true otherwise.  If
 * @delay is zero and @dwork is idle, it will be scheduled for immediate
 * execution.
 */
bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
                           struct delayed_work *dwork, unsigned long delay)
{
        struct work_struct *work = &dwork->work;
        bool ret = false;
        unsigned long irq_flags;

        /* read the comment in __queue_work() */
        local_irq_save(irq_flags);

        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !clear_pending_if_disabled(work)) {
                __queue_delayed_work(cpu, wq, dwork, delay);
                ret = true;
        }

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL(queue_delayed_work_on);

/**
 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
 * @cpu: CPU number to execute work on
 * @wq: workqueue to use
 * @dwork: work to queue
 * @delay: number of jiffies to wait before queueing
 *
 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
 * modify @dwork's timer so that it expires after @delay.  If @delay is
 * zero, @work is guaranteed to be scheduled immediately regardless of its
 * current state.
 *
 * Return: %false if @dwork was idle and queued, %true if @dwork was
 * pending and its timer was modified.
 *
 * This function is safe to call from any context including IRQ handler.
 * See try_to_grab_pending() for details.
 */
bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
                         struct delayed_work *dwork, unsigned long delay)
{
        unsigned long irq_flags;
        bool ret;

        ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);

        if (!clear_pending_if_disabled(&dwork->work))
                __queue_delayed_work(cpu, wq, dwork, delay);

        local_irq_restore(irq_flags);
        return ret;
}
EXPORT_SYMBOL_GPL(mod_delayed_work_on);

static void rcu_work_rcufn(struct rcu_head *rcu)
{
        struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);

        /* read the comment in __queue_work() */
        local_irq_disable();
        __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
        local_irq_enable();
}

/**
 * queue_rcu_work - queue work after a RCU grace period
 * @wq: workqueue to use
 * @rwork: work to queue
 *
 * Return: %false if @rwork was already pending, %true otherwise.  Note
 * that a full RCU grace period is guaranteed only after a %true return.
 * While @rwork is guaranteed to be executed after a %false return, the
 * execution may happen before a full RCU grace period has passed.
 */
bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
{
        struct work_struct *work = &rwork->work;

        /*
         * rcu_work can't be canceled or disabled. Warn if the user reached
         * inside @rwork and disabled the inner work.
         */
        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
            !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
                rwork->wq = wq;
                call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
                return true;
        }

        return false;
}
EXPORT_SYMBOL(queue_rcu_work);

static struct worker *alloc_worker(int node)
{
        struct worker *worker;

        worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
        if (worker) {
                INIT_LIST_HEAD(&worker->entry);
                INIT_LIST_HEAD(&worker->scheduled);
                INIT_LIST_HEAD(&worker->node);
                /* on creation a worker is in !idle && prep state */
                worker->flags = WORKER_PREP;
        }
        return worker;
}

static cpumask_t *pool_allowed_cpus(struct worker_pool *pool)
{
        if (pool->cpu < 0 && pool->attrs->affn_strict)
                return pool->attrs->__pod_cpumask;
        else
                return pool->attrs->cpumask;
}

/**
 * worker_attach_to_pool() - attach a worker to a pool
 * @worker: worker to be attached
 * @pool: the target pool
 *
 * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and
 * cpu-binding of @worker are kept coordinated with the pool across
 * cpu-[un]hotplugs.
 */
static void worker_attach_to_pool(struct worker *worker,
                                  struct worker_pool *pool)
{
        mutex_lock(&wq_pool_attach_mutex);

        /*
         * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable
         * across this function. See the comments above the flag definition for
         * details. BH workers are, while per-CPU, always DISASSOCIATED.
         */
        if (pool->flags & POOL_DISASSOCIATED) {
                worker->flags |= WORKER_UNBOUND;
        } else {
                WARN_ON_ONCE(pool->flags & POOL_BH);
                kthread_set_per_cpu(worker->task, pool->cpu);
        }

        if (worker->rescue_wq)
                set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool));

        list_add_tail(&worker->node, &pool->workers);
        worker->pool = pool;

        mutex_unlock(&wq_pool_attach_mutex);
}

static void unbind_worker(struct worker *worker)
{
        lockdep_assert_held(&wq_pool_attach_mutex);

        kthread_set_per_cpu(worker->task, -1);
        if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
        else
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
}


static void detach_worker(struct worker *worker)
{
        lockdep_assert_held(&wq_pool_attach_mutex);

        unbind_worker(worker);
        list_del(&worker->node);
}

/**
 * worker_detach_from_pool() - detach a worker from its pool
 * @worker: worker which is attached to its pool
 *
 * Undo the attaching which had been done in worker_attach_to_pool().  The
 * caller worker shouldn't access to the pool after detached except it has
 * other reference to the pool.
 */
static void worker_detach_from_pool(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        /* there is one permanent BH worker per CPU which should never detach */
        WARN_ON_ONCE(pool->flags & POOL_BH);

        mutex_lock(&wq_pool_attach_mutex);
        detach_worker(worker);
        worker->pool = NULL;
        mutex_unlock(&wq_pool_attach_mutex);

        /* clear leftover flags without pool->lock after it is detached */
        worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
}

static int format_worker_id(char *buf, size_t size, struct worker *worker,
                            struct worker_pool *pool)
{
        if (worker->rescue_wq)
                return scnprintf(buf, size, "kworker/R-%s",
                                 worker->rescue_wq->name);

        if (pool) {
                if (pool->cpu >= 0)
                        return scnprintf(buf, size, "kworker/%d:%d%s",
                                         pool->cpu, worker->id,
                                         pool->attrs->nice < 0  ? "H" : "");
                else
                        return scnprintf(buf, size, "kworker/u%d:%d",
                                         pool->id, worker->id);
        } else {
                return scnprintf(buf, size, "kworker/dying");
        }
}

/**
 * create_worker - create a new workqueue worker
 * @pool: pool the new worker will belong to
 *
 * Create and start a new worker which is attached to @pool.
 *
 * CONTEXT:
 * Might sleep.  Does GFP_KERNEL allocations.
 *
 * Return:
 * Pointer to the newly created worker.
 */
static struct worker *create_worker(struct worker_pool *pool)
{
        struct worker *worker;
        int id;

        /* ID is needed to determine kthread name */
        id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
        if (id < 0) {
                pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",
                            ERR_PTR(id));
                return NULL;
        }

        worker = alloc_worker(pool->node);
        if (!worker) {
                pr_err_once("workqueue: Failed to allocate a worker\n");
                goto fail;
        }

        worker->id = id;

        if (!(pool->flags & POOL_BH)) {
                char id_buf[WORKER_ID_LEN];

                format_worker_id(id_buf, sizeof(id_buf), worker, pool);
                worker->task = kthread_create_on_node(worker_thread, worker,
                                                      pool->node, "%s", id_buf);
                if (IS_ERR(worker->task)) {
                        if (PTR_ERR(worker->task) == -EINTR) {
                                pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",
                                       id_buf);
                        } else {
                                pr_err_once("workqueue: Failed to create a worker thread: %pe",
                                            worker->task);
                        }
                        goto fail;
                }

                set_user_nice(worker->task, pool->attrs->nice);
                kthread_bind_mask(worker->task, pool_allowed_cpus(pool));
        }

        /* successful, attach the worker to the pool */
        worker_attach_to_pool(worker, pool);

        /* start the newly created worker */
        raw_spin_lock_irq(&pool->lock);

        worker->pool->nr_workers++;
        worker_enter_idle(worker);

        /*
         * @worker is waiting on a completion in kthread() and will trigger hung
         * check if not woken up soon. As kick_pool() is noop if @pool is empty,
         * wake it up explicitly.
         */
        if (worker->task)
                wake_up_process(worker->task);

        raw_spin_unlock_irq(&pool->lock);

        return worker;

fail:
        ida_free(&pool->worker_ida, id);
        kfree(worker);
        return NULL;
}

static void detach_dying_workers(struct list_head *cull_list)
{
        struct worker *worker;

        list_for_each_entry(worker, cull_list, entry)
                detach_worker(worker);
}

static void reap_dying_workers(struct list_head *cull_list)
{
        struct worker *worker, *tmp;

        list_for_each_entry_safe(worker, tmp, cull_list, entry) {
                list_del_init(&worker->entry);
                kthread_stop_put(worker->task);
                kfree(worker);
        }
}

/**
 * set_worker_dying - Tag a worker for destruction
 * @worker: worker to be destroyed
 * @list: transfer worker away from its pool->idle_list and into list
 *
 * Tag @worker for destruction and adjust @pool stats accordingly.  The worker
 * should be idle.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void set_worker_dying(struct worker *worker, struct list_head *list)
{
        struct worker_pool *pool = worker->pool;

        lockdep_assert_held(&pool->lock);
        lockdep_assert_held(&wq_pool_attach_mutex);

        /* sanity check frenzy */
        if (WARN_ON(worker->current_work) ||
            WARN_ON(!list_empty(&worker->scheduled)) ||
            WARN_ON(!(worker->flags & WORKER_IDLE)))
                return;

        pool->nr_workers--;
        pool->nr_idle--;

        worker->flags |= WORKER_DIE;

        list_move(&worker->entry, list);

        /* get an extra task struct reference for later kthread_stop_put() */
        get_task_struct(worker->task);
}

/**
 * idle_worker_timeout - check if some idle workers can now be deleted.
 * @t: The pool's idle_timer that just expired
 *
 * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in
 * worker_leave_idle(), as a worker flicking between idle and active while its
 * pool is at the too_many_workers() tipping point would cause too much timer
 * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let
 * it expire and re-evaluate things from there.
 */
static void idle_worker_timeout(struct timer_list *t)
{
        struct worker_pool *pool = timer_container_of(pool, t, idle_timer);
        bool do_cull = false;

        if (work_pending(&pool->idle_cull_work))
                return;

        raw_spin_lock_irq(&pool->lock);

        if (too_many_workers(pool)) {
                struct worker *worker;
                unsigned long expires;

                /* idle_list is kept in LIFO order, check the last one */
                worker = list_last_entry(&pool->idle_list, struct worker, entry);
                expires = worker->last_active + IDLE_WORKER_TIMEOUT;
                do_cull = !time_before(jiffies, expires);

                if (!do_cull)
                        mod_timer(&pool->idle_timer, expires);
        }
        raw_spin_unlock_irq(&pool->lock);

        if (do_cull)
                queue_work(system_unbound_wq, &pool->idle_cull_work);
}

/**
 * idle_cull_fn - cull workers that have been idle for too long.
 * @work: the pool's work for handling these idle workers
 *
 * This goes through a pool's idle workers and gets rid of those that have been
 * idle for at least IDLE_WORKER_TIMEOUT seconds.
 *
 * We don't want to disturb isolated CPUs because of a pcpu kworker being
 * culled, so this also resets worker affinity. This requires a sleepable
 * context, hence the split between timer callback and work item.
 */
static void idle_cull_fn(struct work_struct *work)
{
        struct worker_pool *pool = container_of(work, struct worker_pool, idle_cull_work);
        LIST_HEAD(cull_list);

        /*
         * Grabbing wq_pool_attach_mutex here ensures an already-running worker
         * cannot proceed beyong set_pf_worker() in its self-destruct path.
         * This is required as a previously-preempted worker could run after
         * set_worker_dying() has happened but before detach_dying_workers() did.
         */
        mutex_lock(&wq_pool_attach_mutex);
        raw_spin_lock_irq(&pool->lock);

        while (too_many_workers(pool)) {
                struct worker *worker;
                unsigned long expires;

                worker = list_last_entry(&pool->idle_list, struct worker, entry);
                expires = worker->last_active + IDLE_WORKER_TIMEOUT;

                if (time_before(jiffies, expires)) {
                        mod_timer(&pool->idle_timer, expires);
                        break;
                }

                set_worker_dying(worker, &cull_list);
        }

        raw_spin_unlock_irq(&pool->lock);
        detach_dying_workers(&cull_list);
        mutex_unlock(&wq_pool_attach_mutex);

        reap_dying_workers(&cull_list);
}

static void send_mayday(struct work_struct *work)
{
        struct pool_workqueue *pwq = get_work_pwq(work);
        struct workqueue_struct *wq = pwq->wq;

        lockdep_assert_held(&wq_mayday_lock);

        if (!wq->rescuer)
                return;

        /* mayday mayday mayday */
        if (list_empty(&pwq->mayday_node)) {
                /*
                 * If @pwq is for an unbound wq, its base ref may be put at
                 * any time due to an attribute change.  Pin @pwq until the
                 * rescuer is done with it.
                 */
                get_pwq(pwq);
                list_add_tail(&pwq->mayday_node, &wq->maydays);
                wake_up_process(wq->rescuer->task);
                pwq->stats[PWQ_STAT_MAYDAY]++;
        }
}

static void pool_mayday_timeout(struct timer_list *t)
{
        struct worker_pool *pool = timer_container_of(pool, t, mayday_timer);
        struct work_struct *work;

        raw_spin_lock_irq(&pool->lock);
        raw_spin_lock(&wq_mayday_lock);                /* for wq->maydays */

        if (need_to_create_worker(pool)) {
                /*
                 * We've been trying to create a new worker but
                 * haven't been successful.  We might be hitting an
                 * allocation deadlock.  Send distress signals to
                 * rescuers.
                 */
                list_for_each_entry(work, &pool->worklist, entry)
                        send_mayday(work);
        }

        raw_spin_unlock(&wq_mayday_lock);
        raw_spin_unlock_irq(&pool->lock);

        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
}

/**
 * maybe_create_worker - create a new worker if necessary
 * @pool: pool to create a new worker for
 *
 * Create a new worker for @pool if necessary.  @pool is guaranteed to
 * have at least one idle worker on return from this function.  If
 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
 * sent to all rescuers with works scheduled on @pool to resolve
 * possible allocation deadlock.
 *
 * On return, need_to_create_worker() is guaranteed to be %false and
 * may_start_working() %true.
 *
 * LOCKING:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.  Called only from
 * manager.
 */
static void maybe_create_worker(struct worker_pool *pool)
__releases(&pool->lock)
__acquires(&pool->lock)
{
restart:
        raw_spin_unlock_irq(&pool->lock);

        /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);

        while (true) {
                if (create_worker(pool) || !need_to_create_worker(pool))
                        break;

                schedule_timeout_interruptible(CREATE_COOLDOWN);

                if (!need_to_create_worker(pool))
                        break;
        }

        timer_delete_sync(&pool->mayday_timer);
        raw_spin_lock_irq(&pool->lock);
        /*
         * This is necessary even after a new worker was just successfully
         * created as @pool->lock was dropped and the new worker might have
         * already become busy.
         */
        if (need_to_create_worker(pool))
                goto restart;
}

/**
 * manage_workers - manage worker pool
 * @worker: self
 *
 * Assume the manager role and manage the worker pool @worker belongs
 * to.  At any given time, there can be only zero or one manager per
 * pool.  The exclusion is handled automatically by this function.
 *
 * The caller can safely start processing works on false return.  On
 * true return, it's guaranteed that need_to_create_worker() is false
 * and may_start_working() is true.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.  Does GFP_KERNEL allocations.
 *
 * Return:
 * %false if the pool doesn't need management and the caller can safely
 * start processing works, %true if management function was performed and
 * the conditions that the caller verified before calling the function may
 * no longer be true.
 */
static bool manage_workers(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (pool->flags & POOL_MANAGER_ACTIVE)
                return false;

        pool->flags |= POOL_MANAGER_ACTIVE;
        pool->manager = worker;

        maybe_create_worker(pool);

        pool->manager = NULL;
        pool->flags &= ~POOL_MANAGER_ACTIVE;
        rcuwait_wake_up(&manager_wait);
        return true;
}

/**
 * process_one_work - process single work
 * @worker: self
 * @work: work to process
 *
 * Process @work.  This function contains all the logics necessary to
 * process a single work including synchronization against and
 * interaction with other workers on the same cpu, queueing and
 * flushing.  As long as context requirement is met, any worker can
 * call this function to process a work.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
 */
static void process_one_work(struct worker *worker, struct work_struct *work)
__releases(&pool->lock)
__acquires(&pool->lock)
{
        struct pool_workqueue *pwq = get_work_pwq(work);
        struct worker_pool *pool = worker->pool;
        unsigned long work_data;
        int lockdep_start_depth, rcu_start_depth;
        bool bh_draining = pool->flags & POOL_BH_DRAINING;
#ifdef CONFIG_LOCKDEP
        /*
         * It is permissible to free the struct work_struct from
         * inside the function that is called from it, this we need to
         * take into account for lockdep too.  To avoid bogus "held
         * lock freed" warnings as well as problems when looking into
         * work->lockdep_map, make a copy and use that here.
         */
        struct lockdep_map lockdep_map;

        lockdep_copy_map(&lockdep_map, &work->lockdep_map);
#endif
        /* ensure we're on the correct CPU */
        WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
                     raw_smp_processor_id() != pool->cpu);

        /* claim and dequeue */
        debug_work_deactivate(work);
        hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
        worker->current_work = work;
        worker->current_func = work->func;
        worker->current_pwq = pwq;
        if (worker->task)
                worker->current_at = worker->task->se.sum_exec_runtime;
        work_data = *work_data_bits(work);
        worker->current_color = get_work_color(work_data);

        /*
         * Record wq name for cmdline and debug reporting, may get
         * overridden through set_worker_desc().
         */
        strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);

        list_del_init(&work->entry);

        /*
         * CPU intensive works don't participate in concurrency management.
         * They're the scheduler's responsibility.  This takes @worker out
         * of concurrency management and the next code block will chain
         * execution of the pending work items.
         */
        if (unlikely(pwq->wq->flags & WQ_CPU_INTENSIVE))
                worker_set_flags(worker, WORKER_CPU_INTENSIVE);

        /*
         * Kick @pool if necessary. It's always noop for per-cpu worker pools
         * since nr_running would always be >= 1 at this point. This is used to
         * chain execution of the pending work items for WORKER_NOT_RUNNING
         * workers such as the UNBOUND and CPU_INTENSIVE ones.
         */
        kick_pool(pool);

        /*
         * Record the last pool and clear PENDING which should be the last
         * update to @work.  Also, do this inside @pool->lock so that
         * PENDING and queued state changes happen together while IRQ is
         * disabled.
         */
        set_work_pool_and_clear_pending(work, pool->id, pool_offq_flags(pool));

        pwq->stats[PWQ_STAT_STARTED]++;
        raw_spin_unlock_irq(&pool->lock);

        rcu_start_depth = rcu_preempt_depth();
        lockdep_start_depth = lockdep_depth(current);
        /* see drain_dead_softirq_workfn() */
        if (!bh_draining)
                lock_map_acquire(pwq->wq->lockdep_map);
        lock_map_acquire(&lockdep_map);
        /*
         * Strictly speaking we should mark the invariant state without holding
         * any locks, that is, before these two lock_map_acquire()'s.
         *
         * However, that would result in:
         *
         *   A(W1)
         *   WFC(C)
         *                A(W1)
         *                C(C)
         *
         * Which would create W1->C->W1 dependencies, even though there is no
         * actual deadlock possible. There are two solutions, using a
         * read-recursive acquire on the work(queue) 'locks', but this will then
         * hit the lockdep limitation on recursive locks, or simply discard
         * these locks.
         *
         * AFAICT there is no possible deadlock scenario between the
         * flush_work() and complete() primitives (except for single-threaded
         * workqueues), so hiding them isn't a problem.
         */
        lockdep_invariant_state(true);
        trace_workqueue_execute_start(work);
        worker->current_func(work);
        /*
         * While we must be careful to not use "work" after this, the trace
         * point will only record its address.
         */
        trace_workqueue_execute_end(work, worker->current_func);

        lock_map_release(&lockdep_map);
        if (!bh_draining)
                lock_map_release(pwq->wq->lockdep_map);

        if (unlikely((worker->task && in_atomic()) ||
                     lockdep_depth(current) != lockdep_start_depth ||
                     rcu_preempt_depth() != rcu_start_depth)) {
                pr_err("BUG: workqueue leaked atomic, lock or RCU: %s[%d]\n"
                       "     preempt=0x%08x lock=%d->%d RCU=%d->%d workfn=%ps\n",
                       current->comm, task_pid_nr(current), preempt_count(),
                       lockdep_start_depth, lockdep_depth(current),
                       rcu_start_depth, rcu_preempt_depth(),
                       worker->current_func);
                debug_show_held_locks(current);
                dump_stack();
        }

        /*
         * The following prevents a kworker from hogging CPU on !PREEMPTION
         * kernels, where a requeueing work item waiting for something to
         * happen could deadlock with stop_machine as such work item could
         * indefinitely requeue itself while all other CPUs are trapped in
         * stop_machine. At the same time, report a quiescent RCU state so
         * the same condition doesn't freeze RCU.
         */
        if (worker->task)
                cond_resched();

        raw_spin_lock_irq(&pool->lock);

        pwq->stats[PWQ_STAT_COMPLETED]++;

        /*
         * In addition to %WQ_CPU_INTENSIVE, @worker may also have been marked
         * CPU intensive by wq_worker_tick() if @work hogged CPU longer than
         * wq_cpu_intensive_thresh_us. Clear it.
         */
        worker_clr_flags(worker, WORKER_CPU_INTENSIVE);

        /* tag the worker for identification in schedule() */
        worker->last_func = worker->current_func;

        /* we're done with it, release */
        hash_del(&worker->hentry);
        worker->current_work = NULL;
        worker->current_func = NULL;
        worker->current_pwq = NULL;
        worker->current_color = INT_MAX;

        /* must be the last step, see the function comment */
        pwq_dec_nr_in_flight(pwq, work_data);
}

/**
 * process_scheduled_works - process scheduled works
 * @worker: self
 *
 * Process all scheduled works.  Please note that the scheduled list
 * may change while processing a work, so this function repeatedly
 * fetches a work from the top and executes it.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
 * multiple times.
 */
static void process_scheduled_works(struct worker *worker)
{
        struct work_struct *work;
        bool first = true;

        while ((work = list_first_entry_or_null(&worker->scheduled,
                                                struct work_struct, entry))) {
                if (first) {
                        worker->pool->watchdog_ts = jiffies;
                        first = false;
                }
                process_one_work(worker, work);
        }
}

static void set_pf_worker(bool val)
{
        mutex_lock(&wq_pool_attach_mutex);
        if (val)
                current->flags |= PF_WQ_WORKER;
        else
                current->flags &= ~PF_WQ_WORKER;
        mutex_unlock(&wq_pool_attach_mutex);
}

/**
 * worker_thread - the worker thread function
 * @__worker: self
 *
 * The worker thread function.  All workers belong to a worker_pool -
 * either a per-cpu one or dynamic unbound one.  These workers process all
 * work items regardless of their specific target workqueue.  The only
 * exception is work items which belong to workqueues with a rescuer which
 * will be explained in rescuer_thread().
 *
 * Return: 0
 */
static int worker_thread(void *__worker)
{
        struct worker *worker = __worker;
        struct worker_pool *pool = worker->pool;

        /* tell the scheduler that this is a workqueue worker */
        set_pf_worker(true);
woke_up:
        raw_spin_lock_irq(&pool->lock);

        /* am I supposed to die? */
        if (unlikely(worker->flags & WORKER_DIE)) {
                raw_spin_unlock_irq(&pool->lock);
                set_pf_worker(false);
                /*
                 * The worker is dead and PF_WQ_WORKER is cleared, worker->pool
                 * shouldn't be accessed, reset it to NULL in case otherwise.
                 */
                worker->pool = NULL;
                ida_free(&pool->worker_ida, worker->id);
                return 0;
        }

        worker_leave_idle(worker);
recheck:
        /* no more worker necessary? */
        if (!need_more_worker(pool))
                goto sleep;

        /* do we need to manage? */
        if (unlikely(!may_start_working(pool)) && manage_workers(worker))
                goto recheck;

        /*
         * ->scheduled list can only be filled while a worker is
         * preparing to process a work or actually processing it.
         * Make sure nobody diddled with it while I was sleeping.
         */
        WARN_ON_ONCE(!list_empty(&worker->scheduled));

        /*
         * Finish PREP stage.  We're guaranteed to have at least one idle
         * worker or that someone else has already assumed the manager
         * role.  This is where @worker starts participating in concurrency
         * management if applicable and concurrency management is restored
         * after being rebound.  See rebind_workers() for details.
         */
        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);

        do {
                struct work_struct *work =
                        list_first_entry(&pool->worklist,
                                         struct work_struct, entry);

                if (assign_work(work, worker, NULL))
                        process_scheduled_works(worker);
        } while (keep_working(pool));

        worker_set_flags(worker, WORKER_PREP);
sleep:
        /*
         * pool->lock is held and there's no work to process and no need to
         * manage, sleep.  Workers are woken up only while holding
         * pool->lock or from local cpu, so setting the current state
         * before releasing pool->lock is enough to prevent losing any
         * event.
         */
        worker_enter_idle(worker);
        __set_current_state(TASK_IDLE);
        raw_spin_unlock_irq(&pool->lock);
        schedule();
        goto woke_up;
}

/**
 * rescuer_thread - the rescuer thread function
 * @__rescuer: self
 *
 * Workqueue rescuer thread function.  There's one rescuer for each
 * workqueue which has WQ_MEM_RECLAIM set.
 *
 * Regular work processing on a pool may block trying to create a new
 * worker which uses GFP_KERNEL allocation which has slight chance of
 * developing into deadlock if some works currently on the same queue
 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
 * the problem rescuer solves.
 *
 * When such condition is possible, the pool summons rescuers of all
 * workqueues which have works queued on the pool and let them process
 * those works so that forward progress can be guaranteed.
 *
 * This should happen rarely.
 *
 * Return: 0
 */
static int rescuer_thread(void *__rescuer)
{
        struct worker *rescuer = __rescuer;
        struct workqueue_struct *wq = rescuer->rescue_wq;
        bool should_stop;

        set_user_nice(current, RESCUER_NICE_LEVEL);

        /*
         * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
         * doesn't participate in concurrency management.
         */
        set_pf_worker(true);
repeat:
        set_current_state(TASK_IDLE);

        /*
         * By the time the rescuer is requested to stop, the workqueue
         * shouldn't have any work pending, but @wq->maydays may still have
         * pwq(s) queued.  This can happen by non-rescuer workers consuming
         * all the work items before the rescuer got to them.  Go through
         * @wq->maydays processing before acting on should_stop so that the
         * list is always empty on exit.
         */
        should_stop = kthread_should_stop();

        /* see whether any pwq is asking for help */
        raw_spin_lock_irq(&wq_mayday_lock);

        while (!list_empty(&wq->maydays)) {
                struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
                                        struct pool_workqueue, mayday_node);
                struct worker_pool *pool = pwq->pool;
                struct work_struct *work, *n;

                __set_current_state(TASK_RUNNING);
                list_del_init(&pwq->mayday_node);

                raw_spin_unlock_irq(&wq_mayday_lock);

                worker_attach_to_pool(rescuer, pool);

                raw_spin_lock_irq(&pool->lock);

                /*
                 * Slurp in all works issued via this workqueue and
                 * process'em.
                 */
                WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
                list_for_each_entry_safe(work, n, &pool->worklist, entry) {
                        if (get_work_pwq(work) == pwq &&
                            assign_work(work, rescuer, &n))
                                pwq->stats[PWQ_STAT_RESCUED]++;
                }

                if (!list_empty(&rescuer->scheduled)) {
                        process_scheduled_works(rescuer);

                        /*
                         * The above execution of rescued work items could
                         * have created more to rescue through
                         * pwq_activate_first_inactive() or chained
                         * queueing.  Let's put @pwq back on mayday list so
                         * that such back-to-back work items, which may be
                         * being used to relieve memory pressure, don't
                         * incur MAYDAY_INTERVAL delay inbetween.
                         */
                        if (pwq->nr_active && need_to_create_worker(pool)) {
                                raw_spin_lock(&wq_mayday_lock);
                                /*
                                 * Queue iff we aren't racing destruction
                                 * and somebody else hasn't queued it already.
                                 */
                                if (wq->rescuer && list_empty(&pwq->mayday_node)) {
                                        get_pwq(pwq);
                                        list_add_tail(&pwq->mayday_node, &wq->maydays);
                                }
                                raw_spin_unlock(&wq_mayday_lock);
                        }
                }

                /*
                 * Leave this pool. Notify regular workers; otherwise, we end up
                 * with 0 concurrency and stalling the execution.
                 */
                kick_pool(pool);

                raw_spin_unlock_irq(&pool->lock);

                worker_detach_from_pool(rescuer);

                /*
                 * Put the reference grabbed by send_mayday().  @pool might
                 * go away any time after it.
                 */
                put_pwq_unlocked(pwq);

                raw_spin_lock_irq(&wq_mayday_lock);
        }

        raw_spin_unlock_irq(&wq_mayday_lock);

        if (should_stop) {
                __set_current_state(TASK_RUNNING);
                set_pf_worker(false);
                return 0;
        }

        /* rescuers should never participate in concurrency management */
        WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
        schedule();
        goto repeat;
}

static void bh_worker(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;
        int nr_restarts = BH_WORKER_RESTARTS;
        unsigned long end = jiffies + BH_WORKER_JIFFIES;

        raw_spin_lock_irq(&pool->lock);
        worker_leave_idle(worker);

        /*
         * This function follows the structure of worker_thread(). See there for
         * explanations on each step.
         */
        if (!need_more_worker(pool))
                goto done;

        WARN_ON_ONCE(!list_empty(&worker->scheduled));
        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);

        do {
                struct work_struct *work =
                        list_first_entry(&pool->worklist,
                                         struct work_struct, entry);

                if (assign_work(work, worker, NULL))
                        process_scheduled_works(worker);
        } while (keep_working(pool) &&
                 --nr_restarts && time_before(jiffies, end));

        worker_set_flags(worker, WORKER_PREP);
done:
        worker_enter_idle(worker);
        kick_pool(pool);
        raw_spin_unlock_irq(&pool->lock);
}

/*
 * TODO: Convert all tasklet users to workqueue and use softirq directly.
 *
 * This is currently called from tasklet[_hi]action() and thus is also called
 * whenever there are tasklets to run. Let's do an early exit if there's nothing
 * queued. Once conversion from tasklet is complete, the need_more_worker() test
 * can be dropped.
 *
 * After full conversion, we'll add worker->softirq_action, directly use the
 * softirq action and obtain the worker pointer from the softirq_action pointer.
 */
void workqueue_softirq_action(bool highpri)
{
        struct worker_pool *pool =
                &per_cpu(bh_worker_pools, smp_processor_id())[highpri];
        if (need_more_worker(pool))
                bh_worker(list_first_entry(&pool->workers, struct worker, node));
}

struct wq_drain_dead_softirq_work {
        struct work_struct        work;
        struct worker_pool        *pool;
        struct completion        done;
};

static void drain_dead_softirq_workfn(struct work_struct *work)
{
        struct wq_drain_dead_softirq_work *dead_work =
                container_of(work, struct wq_drain_dead_softirq_work, work);
        struct worker_pool *pool = dead_work->pool;
        bool repeat;

        /*
         * @pool's CPU is dead and we want to execute its still pending work
         * items from this BH work item which is running on a different CPU. As
         * its CPU is dead, @pool can't be kicked and, as work execution path
         * will be nested, a lockdep annotation needs to be suppressed. Mark
         * @pool with %POOL_BH_DRAINING for the special treatments.
         */
        raw_spin_lock_irq(&pool->lock);
        pool->flags |= POOL_BH_DRAINING;
        raw_spin_unlock_irq(&pool->lock);

        bh_worker(list_first_entry(&pool->workers, struct worker, node));

        raw_spin_lock_irq(&pool->lock);
        pool->flags &= ~POOL_BH_DRAINING;
        repeat = need_more_worker(pool);
        raw_spin_unlock_irq(&pool->lock);

        /*
         * bh_worker() might hit consecutive execution limit and bail. If there
         * still are pending work items, reschedule self and return so that we
         * don't hog this CPU's BH.
         */
        if (repeat) {
                if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                        queue_work(system_bh_highpri_wq, work);
                else
                        queue_work(system_bh_wq, work);
        } else {
                complete(&dead_work->done);
        }
}

/*
 * @cpu is dead. Drain the remaining BH work items on the current CPU. It's
 * possible to allocate dead_work per CPU and avoid flushing. However, then we
 * have to worry about draining overlapping with CPU coming back online or
 * nesting (one CPU's dead_work queued on another CPU which is also dead and so
 * on). Let's keep it simple and drain them synchronously. These are BH work
 * items which shouldn't be requeued on the same pool. Shouldn't take long.
 */
void workqueue_softirq_dead(unsigned int cpu)
{
        int i;

        for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                struct worker_pool *pool = &per_cpu(bh_worker_pools, cpu)[i];
                struct wq_drain_dead_softirq_work dead_work;

                if (!need_more_worker(pool))
                        continue;

                INIT_WORK_ONSTACK(&dead_work.work, drain_dead_softirq_workfn);
                dead_work.pool = pool;
                init_completion(&dead_work.done);

                if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)
                        queue_work(system_bh_highpri_wq, &dead_work.work);
                else
                        queue_work(system_bh_wq, &dead_work.work);

                wait_for_completion(&dead_work.done);
                destroy_work_on_stack(&dead_work.work);
        }
}

/**
 * check_flush_dependency - check for flush dependency sanity
 * @target_wq: workqueue being flushed
 * @target_work: work item being flushed (NULL for workqueue flushes)
 * @from_cancel: are we called from the work cancel path
 *
 * %current is trying to flush the whole @target_wq or @target_work on it.
 * If this is not the cancel path (which implies work being flushed is either
 * already running, or will not be at all), check if @target_wq doesn't have
 * %WQ_MEM_RECLAIM and verify that %current is not reclaiming memory or running
 * on a workqueue which doesn't have %WQ_MEM_RECLAIM as that can break forward-
 * progress guarantee leading to a deadlock.
 */
static void check_flush_dependency(struct workqueue_struct *target_wq,
                                   struct work_struct *target_work,
                                   bool from_cancel)
{
        work_func_t target_func;
        struct worker *worker;

        if (from_cancel || target_wq->flags & WQ_MEM_RECLAIM)
                return;

        worker = current_wq_worker();
        target_func = target_work ? target_work->func : NULL;

        WARN_ONCE(current->flags & PF_MEMALLOC,
                  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
                  current->pid, current->comm, target_wq->name, target_func);
        WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
                              (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
                  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
                  worker->current_pwq->wq->name, worker->current_func,
                  target_wq->name, target_func);
}

struct wq_barrier {
        struct work_struct        work;
        struct completion        done;
        struct task_struct        *task;        /* purely informational */
};

static void wq_barrier_func(struct work_struct *work)
{
        struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
        complete(&barr->done);
}

/**
 * insert_wq_barrier - insert a barrier work
 * @pwq: pwq to insert barrier into
 * @barr: wq_barrier to insert
 * @target: target work to attach @barr to
 * @worker: worker currently executing @target, NULL if @target is not executing
 *
 * @barr is linked to @target such that @barr is completed only after
 * @target finishes execution.  Please note that the ordering
 * guarantee is observed only with respect to @target and on the local
 * cpu.
 *
 * Currently, a queued barrier can't be canceled.  This is because
 * try_to_grab_pending() can't determine whether the work to be
 * grabbed is at the head of the queue and thus can't clear LINKED
 * flag of the previous work while there must be a valid next work
 * after a work with LINKED flag set.
 *
 * Note that when @worker is non-NULL, @target may be modified
 * underneath us, so we can't reliably determine pwq from @target.
 *
 * CONTEXT:
 * raw_spin_lock_irq(pool->lock).
 */
static void insert_wq_barrier(struct pool_workqueue *pwq,
                              struct wq_barrier *barr,
                              struct work_struct *target, struct worker *worker)
{
        static __maybe_unused struct lock_class_key bh_key, thr_key;
        unsigned int work_flags = 0;
        unsigned int work_color;
        struct list_head *head;

        /*
         * debugobject calls are safe here even with pool->lock locked
         * as we know for sure that this will not trigger any of the
         * checks and call back into the fixup functions where we
         * might deadlock.
         *
         * BH and threaded workqueues need separate lockdep keys to avoid
         * spuriously triggering "inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W}
         * usage".
         */
        INIT_WORK_ONSTACK_KEY(&barr->work, wq_barrier_func,
                              (pwq->wq->flags & WQ_BH) ? &bh_key : &thr_key);
        __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));

        init_completion_map(&barr->done, &target->lockdep_map);

        barr->task = current;

        /* The barrier work item does not participate in nr_active. */
        work_flags |= WORK_STRUCT_INACTIVE;

        /*
         * If @target is currently being executed, schedule the
         * barrier to the worker; otherwise, put it after @target.
         */
        if (worker) {
                head = worker->scheduled.next;
                work_color = worker->current_color;
        } else {
                unsigned long *bits = work_data_bits(target);

                head = target->entry.next;
                /* there can already be other linked works, inherit and set */
                work_flags |= *bits & WORK_STRUCT_LINKED;
                work_color = get_work_color(*bits);
                __set_bit(WORK_STRUCT_LINKED_BIT, bits);
        }

        pwq->nr_in_flight[work_color]++;
        work_flags |= work_color_to_flags(work_color);

        insert_work(pwq, &barr->work, head, work_flags);
}

/**
 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
 * @wq: workqueue being flushed
 * @flush_color: new flush color, < 0 for no-op
 * @work_color: new work color, < 0 for no-op
 *
 * Prepare pwqs for workqueue flushing.
 *
 * If @flush_color is non-negative, flush_color on all pwqs should be
 * -1.  If no pwq has in-flight commands at the specified color, all
 * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
 * has in flight commands, its pwq->flush_color is set to
 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
 * wakeup logic is armed and %true is returned.
 *
 * The caller should have initialized @wq->first_flusher prior to
 * calling this function with non-negative @flush_color.  If
 * @flush_color is negative, no flush color update is done and %false
 * is returned.
 *
 * If @work_color is non-negative, all pwqs should have the same
 * work_color which is previous to @work_color and all will be
 * advanced to @work_color.
 *
 * CONTEXT:
 * mutex_lock(wq->mutex).
 *
 * Return:
 * %true if @flush_color >= 0 and there's something to flush.  %false
 * otherwise.
 */
static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
                                      int flush_color, int work_color)
{
        bool wait = false;
        struct pool_workqueue *pwq;
        struct worker_pool *current_pool = NULL;

        if (flush_color >= 0) {
                WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
                atomic_set(&wq->nr_pwqs_to_flush, 1);
        }

        /*
         * For unbound workqueue, pwqs will map to only a few pools.
         * Most of the time, pwqs within the same pool will be linked
         * sequentially to wq->pwqs by cpu index. So in the majority
         * of pwq iters, the pool is the same, only doing lock/unlock
         * if the pool has changed. This can largely reduce expensive
         * lock operations.
         */
        for_each_pwq(pwq, wq) {
                if (current_pool != pwq->pool) {
                        if (likely(current_pool))
                                raw_spin_unlock_irq(&current_pool->lock);
                        current_pool = pwq->pool;
                        raw_spin_lock_irq(&current_pool->lock);
                }

                if (flush_color >= 0) {
                        WARN_ON_ONCE(pwq->flush_color != -1);

                        if (pwq->nr_in_flight[flush_color]) {
                                pwq->flush_color = flush_color;
                                atomic_inc(&wq->nr_pwqs_to_flush);
                                wait = true;
                        }
                }

                if (work_color >= 0) {
                        WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
                        pwq->work_color = work_color;
                }

        }

        if (current_pool)
                raw_spin_unlock_irq(&current_pool->lock);

        if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
                complete(&wq->first_flusher->done);

        return wait;
}

static void touch_wq_lockdep_map(struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
        if (unlikely(!wq->lockdep_map))
                return;

        if (wq->flags & WQ_BH)
                local_bh_disable();

        lock_map_acquire(wq->lockdep_map);
        lock_map_release(wq->lockdep_map);

        if (wq->flags & WQ_BH)
                local_bh_enable();
#endif
}

static void touch_work_lockdep_map(struct work_struct *work,
                                   struct workqueue_struct *wq)
{
#ifdef CONFIG_LOCKDEP
        if (wq->flags & WQ_BH)
                local_bh_disable();

        lock_map_acquire(&work->lockdep_map);
        lock_map_release(&work->lockdep_map);

        if (wq->flags & WQ_BH)
                local_bh_enable();
#endif
}

/**
 * __flush_workqueue - ensure that any scheduled work has run to completion.
 * @wq: workqueue to flush
 *
 * This function sleeps until all work items which were queued on entry
 * have finished execution, but it is not livelocked by new incoming ones.
 */
void __flush_workqueue(struct workqueue_struct *wq)
{
        struct wq_flusher this_flusher = {
                .list = LIST_HEAD_INIT(this_flusher.list),
                .flush_color = -1,
                .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, (*wq->lockdep_map)),
        };
        int next_color;

        if (WARN_ON(!wq_online))
                return;

        touch_wq_lockdep_map(wq);

        mutex_lock(&wq->mutex);

        /*
         * Start-to-wait phase
         */
        next_color = work_next_color(wq->work_color);

        if (next_color != wq->flush_color) {
                /*
                 * Color space is not full.  The current work_color
                 * becomes our flush_color and work_color is advanced
                 * by one.
                 */
                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
                this_flusher.flush_color = wq->work_color;
                wq->work_color = next_color;

                if (!wq->first_flusher) {
                        /* no flush in progress, become the first flusher */
                        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);

                        wq->first_flusher = &this_flusher;

                        if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
                                                       wq->work_color)) {
                                /* nothing to flush, done */
                                wq->flush_color = next_color;
                                wq->first_flusher = NULL;
                                goto out_unlock;
                        }
                } else {
                        /* wait in queue */
                        WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
                        list_add_tail(&this_flusher.list, &wq->flusher_queue);
                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
                }
        } else {
                /*
                 * Oops, color space is full, wait on overflow queue.
                 * The next flush completion will assign us
                 * flush_color and transfer to flusher_queue.
                 */
                list_add_tail(&this_flusher.list, &wq->flusher_overflow);
        }

        check_flush_dependency(wq, NULL, false);

        mutex_unlock(&wq->mutex);

        wait_for_completion(&this_flusher.done);

        /*
         * Wake-up-and-cascade phase
         *
         * First flushers are responsible for cascading flushes and
         * handling overflow.  Non-first flushers can simply return.
         */
        if (READ_ONCE(wq->first_flusher) != &this_flusher)
                return;

        mutex_lock(&wq->mutex);

        /* we might have raced, check again with mutex held */
        if (wq->first_flusher != &this_flusher)
                goto out_unlock;

        WRITE_ONCE(wq->first_flusher, NULL);

        WARN_ON_ONCE(!list_empty(&this_flusher.list));
        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);

        while (true) {
                struct wq_flusher *next, *tmp;

                /* complete all the flushers sharing the current flush color */
                list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
                        if (next->flush_color != wq->flush_color)
                                break;
                        list_del_init(&next->list);
                        complete(&next->done);
                }

                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
                             wq->flush_color != work_next_color(wq->work_color));

                /* this flush_color is finished, advance by one */
                wq->flush_color = work_next_color(wq->flush_color);

                /* one color has been freed, handle overflow queue */
                if (!list_empty(&wq->flusher_overflow)) {
                        /*
                         * Assign the same color to all overflowed
                         * flushers, advance work_color and append to
                         * flusher_queue.  This is the start-to-wait
                         * phase for these overflowed flushers.
                         */
                        list_for_each_entry(tmp, &wq->flusher_overflow, list)
                                tmp->flush_color = wq->work_color;

                        wq->work_color = work_next_color(wq->work_color);

                        list_splice_tail_init(&wq->flusher_overflow,
                                              &wq->flusher_queue);
                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
                }

                if (list_empty(&wq->flusher_queue)) {
                        WARN_ON_ONCE(wq->flush_color != wq->work_color);
                        break;
                }

                /*
                 * Need to flush more colors.  Make the next flusher
                 * the new first flusher and arm pwqs.
                 */
                WARN_ON_ONCE(wq->flush_color == wq->work_color);
                WARN_ON_ONCE(wq->flush_color != next->flush_color);

                list_del_init(&next->list);
                wq->first_flusher = next;

                if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
                        break;

                /*
                 * Meh... this color is already done, clear first
                 * flusher and repeat cascading.
                 */
                wq->first_flusher = NULL;
        }

out_unlock:
        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL(__flush_workqueue);

/**
 * drain_workqueue - drain a workqueue
 * @wq: workqueue to drain
 *
 * Wait until the workqueue becomes empty.  While draining is in progress,
 * only chain queueing is allowed.  IOW, only currently pending or running
 * work items on @wq can queue further work items on it.  @wq is flushed
 * repeatedly until it becomes empty.  The number of flushing is determined
 * by the depth of chaining and should be relatively short.  Whine if it
 * takes too long.
 */
void drain_workqueue(struct workqueue_struct *wq)
{
        unsigned int flush_cnt = 0;
        struct pool_workqueue *pwq;

        /*
         * __queue_work() needs to test whether there are drainers, is much
         * hotter than drain_workqueue() and already looks at @wq->flags.
         * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
         */
        mutex_lock(&wq->mutex);
        if (!wq->nr_drainers++)
                wq->flags |= __WQ_DRAINING;
        mutex_unlock(&wq->mutex);
reflush:
        __flush_workqueue(wq);

        mutex_lock(&wq->mutex);

        for_each_pwq(pwq, wq) {
                bool drained;

                raw_spin_lock_irq(&pwq->pool->lock);
                drained = pwq_is_empty(pwq);
                raw_spin_unlock_irq(&pwq->pool->lock);

                if (drained)
                        continue;

                if (++flush_cnt == 10 ||
                    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
                        pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
                                wq->name, __func__, flush_cnt);

                mutex_unlock(&wq->mutex);
                goto reflush;
        }

        if (!--wq->nr_drainers)
                wq->flags &= ~__WQ_DRAINING;
        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(drain_workqueue);

static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
                             bool from_cancel)
{
        struct worker *worker = NULL;
        struct worker_pool *pool;
        struct pool_workqueue *pwq;
        struct workqueue_struct *wq;

        rcu_read_lock();
        pool = get_work_pool(work);
        if (!pool) {
                rcu_read_unlock();
                return false;
        }

        raw_spin_lock_irq(&pool->lock);
        /* see the comment in try_to_grab_pending() with the same code */
        pwq = get_work_pwq(work);
        if (pwq) {
                if (unlikely(pwq->pool != pool))
                        goto already_gone;
        } else {
                worker = find_worker_executing_work(pool, work);
                if (!worker)
                        goto already_gone;
                pwq = worker->current_pwq;
        }

        wq = pwq->wq;
        check_flush_dependency(wq, work, from_cancel);

        insert_wq_barrier(pwq, barr, work, worker);
        raw_spin_unlock_irq(&pool->lock);

        touch_work_lockdep_map(work, wq);

        /*
         * Force a lock recursion deadlock when using flush_work() inside a
         * single-threaded or rescuer equipped workqueue.
         *
         * For single threaded workqueues the deadlock happens when the work
         * is after the work issuing the flush_work(). For rescuer equipped
         * workqueues the deadlock happens when the rescuer stalls, blocking
         * forward progress.
         */
        if (!from_cancel && (wq->saved_max_active == 1 || wq->rescuer))
                touch_wq_lockdep_map(wq);

        rcu_read_unlock();
        return true;
already_gone:
        raw_spin_unlock_irq(&pool->lock);
        rcu_read_unlock();
        return false;
}

static bool __flush_work(struct work_struct *work, bool from_cancel)
{
        struct wq_barrier barr;

        if (WARN_ON(!wq_online))
                return false;

        if (WARN_ON(!work->func))
                return false;

        if (!start_flush_work(work, &barr, from_cancel))
                return false;

        /*
         * start_flush_work() returned %true. If @from_cancel is set, we know
         * that @work must have been executing during start_flush_work() and
         * can't currently be queued. Its data must contain OFFQ bits. If @work
         * was queued on a BH workqueue, we also know that it was running in the
         * BH context and thus can be busy-waited.
         */
        if (from_cancel) {
                unsigned long data = *work_data_bits(work);

                if (!WARN_ON_ONCE(data & WORK_STRUCT_PWQ) &&
                    (data & WORK_OFFQ_BH)) {
                        /*
                         * On RT, prevent a live lock when %current preempted
                         * soft interrupt processing or prevents ksoftirqd from
                         * running by keeping flipping BH. If the BH work item
                         * runs on a different CPU then this has no effect other
                         * than doing the BH disable/enable dance for nothing.
                         * This is copied from
                         * kernel/softirq.c::tasklet_unlock_spin_wait().
                         */
                        while (!try_wait_for_completion(&barr.done)) {
                                if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
                                        local_bh_disable();
                                        local_bh_enable();
                                } else {
                                        cpu_relax();
                                }
                        }
                        goto out_destroy;
                }
        }

        wait_for_completion(&barr.done);

out_destroy:
        destroy_work_on_stack(&barr.work);
        return true;
}

/**
 * flush_work - wait for a work to finish executing the last queueing instance
 * @work: the work to flush
 *
 * Wait until @work has finished execution.  @work is guaranteed to be idle
 * on return if it hasn't been requeued since flush started.
 *
 * Return:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_work(struct work_struct *work)
{
        might_sleep();
        return __flush_work(work, false);
}
EXPORT_SYMBOL_GPL(flush_work);

/**
 * flush_delayed_work - wait for a dwork to finish executing the last queueing
 * @dwork: the delayed work to flush
 *
 * Delayed timer is cancelled and the pending work is queued for
 * immediate execution.  Like flush_work(), this function only
 * considers the last queueing instance of @dwork.
 *
 * Return:
 * %true if flush_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_delayed_work(struct delayed_work *dwork)
{
        local_irq_disable();
        if (timer_delete_sync(&dwork->timer))
                __queue_work(dwork->cpu, dwork->wq, &dwork->work);
        local_irq_enable();
        return flush_work(&dwork->work);
}
EXPORT_SYMBOL(flush_delayed_work);

/**
 * flush_rcu_work - wait for a rwork to finish executing the last queueing
 * @rwork: the rcu work to flush
 *
 * Return:
 * %true if flush_rcu_work() waited for the work to finish execution,
 * %false if it was already idle.
 */
bool flush_rcu_work(struct rcu_work *rwork)
{
        if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
                rcu_barrier();
                flush_work(&rwork->work);
                return true;
        } else {
                return flush_work(&rwork->work);
        }
}
EXPORT_SYMBOL(flush_rcu_work);

static void work_offqd_disable(struct work_offq_data *offqd)
{
        const unsigned long max = (1lu << WORK_OFFQ_DISABLE_BITS) - 1;

        if (likely(offqd->disable < max))
                offqd->disable++;
        else
                WARN_ONCE(true, "workqueue: work disable count overflowed\n");
}

static void work_offqd_enable(struct work_offq_data *offqd)
{
        if (likely(offqd->disable > 0))
                offqd->disable--;
        else
                WARN_ONCE(true, "workqueue: work disable count underflowed\n");
}

static bool __cancel_work(struct work_struct *work, u32 cflags)
{
        struct work_offq_data offqd;
        unsigned long irq_flags;
        int ret;

        ret = work_grab_pending(work, cflags, &irq_flags);

        work_offqd_unpack(&offqd, *work_data_bits(work));

        if (cflags & WORK_CANCEL_DISABLE)
                work_offqd_disable(&offqd);

        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        local_irq_restore(irq_flags);
        return ret;
}

static bool __cancel_work_sync(struct work_struct *work, u32 cflags)
{
        bool ret;

        ret = __cancel_work(work, cflags | WORK_CANCEL_DISABLE);

        if (*work_data_bits(work) & WORK_OFFQ_BH)
                WARN_ON_ONCE(in_hardirq());
        else
                might_sleep();

        /*
         * Skip __flush_work() during early boot when we know that @work isn't
         * executing. This allows canceling during early boot.
         */
        if (wq_online)
                __flush_work(work, true);

        if (!(cflags & WORK_CANCEL_DISABLE))
                enable_work(work);

        return ret;
}

/*
 * See cancel_delayed_work()
 */
bool cancel_work(struct work_struct *work)
{
        return __cancel_work(work, 0);
}
EXPORT_SYMBOL(cancel_work);

/**
 * cancel_work_sync - cancel a work and wait for it to finish
 * @work: the work to cancel
 *
 * Cancel @work and wait for its execution to finish. This function can be used
 * even if the work re-queues itself or migrates to another workqueue. On return
 * from this function, @work is guaranteed to be not pending or executing on any
 * CPU as long as there aren't racing enqueues.
 *
 * cancel_work_sync(&delayed_work->work) must not be used for delayed_work's.
 * Use cancel_delayed_work_sync() instead.
 *
 * Must be called from a sleepable context if @work was last queued on a non-BH
 * workqueue. Can also be called from non-hardirq atomic contexts including BH
 * if @work was last queued on a BH workqueue.
 *
 * Returns %true if @work was pending, %false otherwise.
 */
bool cancel_work_sync(struct work_struct *work)
{
        return __cancel_work_sync(work, 0);
}
EXPORT_SYMBOL_GPL(cancel_work_sync);

/**
 * cancel_delayed_work - cancel a delayed work
 * @dwork: delayed_work to cancel
 *
 * Kill off a pending delayed_work.
 *
 * Return: %true if @dwork was pending and canceled; %false if it wasn't
 * pending.
 *
 * Note:
 * The work callback function may still be running on return, unless
 * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
 * use cancel_delayed_work_sync() to wait on it.
 *
 * This function is safe to call from any context including IRQ handler.
 */
bool cancel_delayed_work(struct delayed_work *dwork)
{
        return __cancel_work(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work);

/**
 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
 * @dwork: the delayed work cancel
 *
 * This is cancel_work_sync() for delayed works.
 *
 * Return:
 * %true if @dwork was pending, %false otherwise.
 */
bool cancel_delayed_work_sync(struct delayed_work *dwork)
{
        return __cancel_work_sync(&dwork->work, WORK_CANCEL_DELAYED);
}
EXPORT_SYMBOL(cancel_delayed_work_sync);

/**
 * disable_work - Disable and cancel a work item
 * @work: work item to disable
 *
 * Disable @work by incrementing its disable count and cancel it if currently
 * pending. As long as the disable count is non-zero, any attempt to queue @work
 * will fail and return %false. The maximum supported disable depth is 2 to the
 * power of %WORK_OFFQ_DISABLE_BITS, currently 65536.
 *
 * Can be called from any context. Returns %true if @work was pending, %false
 * otherwise.
 */
bool disable_work(struct work_struct *work)
{
        return __cancel_work(work, WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_work);

/**
 * disable_work_sync - Disable, cancel and drain a work item
 * @work: work item to disable
 *
 * Similar to disable_work() but also wait for @work to finish if currently
 * executing.
 *
 * Must be called from a sleepable context if @work was last queued on a non-BH
 * workqueue. Can also be called from non-hardirq atomic contexts including BH
 * if @work was last queued on a BH workqueue.
 *
 * Returns %true if @work was pending, %false otherwise.
 */
bool disable_work_sync(struct work_struct *work)
{
        return __cancel_work_sync(work, WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_work_sync);

/**
 * enable_work - Enable a work item
 * @work: work item to enable
 *
 * Undo disable_work[_sync]() by decrementing @work's disable count. @work can
 * only be queued if its disable count is 0.
 *
 * Can be called from any context. Returns %true if the disable count reached 0.
 * Otherwise, %false.
 */
bool enable_work(struct work_struct *work)
{
        struct work_offq_data offqd;
        unsigned long irq_flags;

        work_grab_pending(work, 0, &irq_flags);

        work_offqd_unpack(&offqd, *work_data_bits(work));
        work_offqd_enable(&offqd);
        set_work_pool_and_clear_pending(work, offqd.pool_id,
                                        work_offqd_pack_flags(&offqd));
        local_irq_restore(irq_flags);

        return !offqd.disable;
}
EXPORT_SYMBOL_GPL(enable_work);

/**
 * disable_delayed_work - Disable and cancel a delayed work item
 * @dwork: delayed work item to disable
 *
 * disable_work() for delayed work items.
 */
bool disable_delayed_work(struct delayed_work *dwork)
{
        return __cancel_work(&dwork->work,
                             WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_delayed_work);

/**
 * disable_delayed_work_sync - Disable, cancel and drain a delayed work item
 * @dwork: delayed work item to disable
 *
 * disable_work_sync() for delayed work items.
 */
bool disable_delayed_work_sync(struct delayed_work *dwork)
{
        return __cancel_work_sync(&dwork->work,
                                  WORK_CANCEL_DELAYED | WORK_CANCEL_DISABLE);
}
EXPORT_SYMBOL_GPL(disable_delayed_work_sync);

/**
 * enable_delayed_work - Enable a delayed work item
 * @dwork: delayed work item to enable
 *
 * enable_work() for delayed work items.
 */
bool enable_delayed_work(struct delayed_work *dwork)
{
        return enable_work(&dwork->work);
}
EXPORT_SYMBOL_GPL(enable_delayed_work);

/**
 * schedule_on_each_cpu - execute a function synchronously on each online CPU
 * @func: the function to call
 *
 * schedule_on_each_cpu() executes @func on each online CPU using the
 * system workqueue and blocks until all CPUs have completed.
 * schedule_on_each_cpu() is very slow.
 *
 * Return:
 * 0 on success, -errno on failure.
 */
int schedule_on_each_cpu(work_func_t func)
{
        int cpu;
        struct work_struct __percpu *works;

        works = alloc_percpu(struct work_struct);
        if (!works)
                return -ENOMEM;

        cpus_read_lock();

        for_each_online_cpu(cpu) {
                struct work_struct *work = per_cpu_ptr(works, cpu);

                INIT_WORK(work, func);
                schedule_work_on(cpu, work);
        }

        for_each_online_cpu(cpu)
                flush_work(per_cpu_ptr(works, cpu));

        cpus_read_unlock();
        free_percpu(works);
        return 0;
}

/**
 * execute_in_process_context - reliably execute the routine with user context
 * @fn:                the function to execute
 * @ew:                guaranteed storage for the execute work structure (must
 *                be available when the work executes)
 *
 * Executes the function immediately if process context is available,
 * otherwise schedules the function for delayed execution.
 *
 * Return:        0 - function was executed
 *                1 - function was scheduled for execution
 */
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
{
        if (!in_interrupt()) {
                fn(&ew->work);
                return 0;
        }

        INIT_WORK(&ew->work, fn);
        schedule_work(&ew->work);

        return 1;
}
EXPORT_SYMBOL_GPL(execute_in_process_context);

/**
 * free_workqueue_attrs - free a workqueue_attrs
 * @attrs: workqueue_attrs to free
 *
 * Undo alloc_workqueue_attrs().
 */
void free_workqueue_attrs(struct workqueue_attrs *attrs)
{
        if (attrs) {
                free_cpumask_var(attrs->cpumask);
                free_cpumask_var(attrs->__pod_cpumask);
                kfree(attrs);
        }
}

/**
 * alloc_workqueue_attrs - allocate a workqueue_attrs
 *
 * Allocate a new workqueue_attrs, initialize with default settings and
 * return it.
 *
 * Return: The allocated new workqueue_attr on success. %NULL on failure.
 */
struct workqueue_attrs *alloc_workqueue_attrs(void)
{
        struct workqueue_attrs *attrs;

        attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
        if (!attrs)
                goto fail;
        if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
                goto fail;
        if (!alloc_cpumask_var(&attrs->__pod_cpumask, GFP_KERNEL))
                goto fail;

        cpumask_copy(attrs->cpumask, cpu_possible_mask);
        attrs->affn_scope = WQ_AFFN_DFL;
        return attrs;
fail:
        free_workqueue_attrs(attrs);
        return NULL;
}

static void copy_workqueue_attrs(struct workqueue_attrs *to,
                                 const struct workqueue_attrs *from)
{
        to->nice = from->nice;
        cpumask_copy(to->cpumask, from->cpumask);
        cpumask_copy(to->__pod_cpumask, from->__pod_cpumask);
        to->affn_strict = from->affn_strict;

        /*
         * Unlike hash and equality test, copying shouldn't ignore wq-only
         * fields as copying is used for both pool and wq attrs. Instead,
         * get_unbound_pool() explicitly clears the fields.
         */
        to->affn_scope = from->affn_scope;
        to->ordered = from->ordered;
}

/*
 * Some attrs fields are workqueue-only. Clear them for worker_pool's. See the
 * comments in 'struct workqueue_attrs' definition.
 */
static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
{
        attrs->affn_scope = WQ_AFFN_NR_TYPES;
        attrs->ordered = false;
        if (attrs->affn_strict)
                cpumask_copy(attrs->cpumask, cpu_possible_mask);
}

/* hash value of the content of @attr */
static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
{
        u32 hash = 0;

        hash = jhash_1word(attrs->nice, hash);
        hash = jhash_1word(attrs->affn_strict, hash);
        hash = jhash(cpumask_bits(attrs->__pod_cpumask),
                     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
        if (!attrs->affn_strict)
                hash = jhash(cpumask_bits(attrs->cpumask),
                             BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
        return hash;
}

/* content equality test */
static bool wqattrs_equal(const struct workqueue_attrs *a,
                          const struct workqueue_attrs *b)
{
        if (a->nice != b->nice)
                return false;
        if (a->affn_strict != b->affn_strict)
                return false;
        if (!cpumask_equal(a->__pod_cpumask, b->__pod_cpumask))
                return false;
        if (!a->affn_strict && !cpumask_equal(a->cpumask, b->cpumask))
                return false;
        return true;
}

/* Update @attrs with actually available CPUs */
static void wqattrs_actualize_cpumask(struct workqueue_attrs *attrs,
                                      const cpumask_t *unbound_cpumask)
{
        /*
         * Calculate the effective CPU mask of @attrs given @unbound_cpumask. If
         * @attrs->cpumask doesn't overlap with @unbound_cpumask, we fallback to
         * @unbound_cpumask.
         */
        cpumask_and(attrs->cpumask, attrs->cpumask, unbound_cpumask);
        if (unlikely(cpumask_empty(attrs->cpumask)))
                cpumask_copy(attrs->cpumask, unbound_cpumask);
}

/* find wq_pod_type to use for @attrs */
static const struct wq_pod_type *
wqattrs_pod_type(const struct workqueue_attrs *attrs)
{
        enum wq_affn_scope scope;
        struct wq_pod_type *pt;

        /* to synchronize access to wq_affn_dfl */
        lockdep_assert_held(&wq_pool_mutex);

        if (attrs->affn_scope == WQ_AFFN_DFL)
                scope = wq_affn_dfl;
        else
                scope = attrs->affn_scope;

        pt = &wq_pod_types[scope];

        if (!WARN_ON_ONCE(attrs->affn_scope == WQ_AFFN_NR_TYPES) &&
            likely(pt->nr_pods))
                return pt;

        /*
         * Before workqueue_init_topology(), only SYSTEM is available which is
         * initialized in workqueue_init_early().
         */
        pt = &wq_pod_types[WQ_AFFN_SYSTEM];
        BUG_ON(!pt->nr_pods);
        return pt;
}

/**
 * init_worker_pool - initialize a newly zalloc'd worker_pool
 * @pool: worker_pool to initialize
 *
 * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs.
 *
 * Return: 0 on success, -errno on failure.  Even on failure, all fields
 * inside @pool proper are initialized and put_unbound_pool() can be called
 * on @pool safely to release it.
 */
static int init_worker_pool(struct worker_pool *pool)
{
        raw_spin_lock_init(&pool->lock);
        pool->id = -1;
        pool->cpu = -1;
        pool->node = NUMA_NO_NODE;
        pool->flags |= POOL_DISASSOCIATED;
        pool->watchdog_ts = jiffies;
        INIT_LIST_HEAD(&pool->worklist);
        INIT_LIST_HEAD(&pool->idle_list);
        hash_init(pool->busy_hash);

        timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
        INIT_WORK(&pool->idle_cull_work, idle_cull_fn);

        timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);

        INIT_LIST_HEAD(&pool->workers);

        ida_init(&pool->worker_ida);
        INIT_HLIST_NODE(&pool->hash_node);
        pool->refcnt = 1;

        /* shouldn't fail above this point */
        pool->attrs = alloc_workqueue_attrs();
        if (!pool->attrs)
                return -ENOMEM;

        wqattrs_clear_for_pool(pool->attrs);

        return 0;
}

#ifdef CONFIG_LOCKDEP
static void wq_init_lockdep(struct workqueue_struct *wq)
{
        char *lock_name;

        lockdep_register_key(&wq->key);
        lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
        if (!lock_name)
                lock_name = wq->name;

        wq->lock_name = lock_name;
        wq->lockdep_map = &wq->__lockdep_map;
        lockdep_init_map(wq->lockdep_map, lock_name, &wq->key, 0);
}

static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
        if (wq->lockdep_map != &wq->__lockdep_map)
                return;

        lockdep_unregister_key(&wq->key);
}

static void wq_free_lockdep(struct workqueue_struct *wq)
{
        if (wq->lockdep_map != &wq->__lockdep_map)
                return;

        if (wq->lock_name != wq->name)
                kfree(wq->lock_name);
}
#else
static void wq_init_lockdep(struct workqueue_struct *wq)
{
}

static void wq_unregister_lockdep(struct workqueue_struct *wq)
{
}

static void wq_free_lockdep(struct workqueue_struct *wq)
{
}
#endif

static void free_node_nr_active(struct wq_node_nr_active **nna_ar)
{
        int node;

        for_each_node(node) {
                kfree(nna_ar[node]);
                nna_ar[node] = NULL;
        }

        kfree(nna_ar[nr_node_ids]);
        nna_ar[nr_node_ids] = NULL;
}

static void init_node_nr_active(struct wq_node_nr_active *nna)
{
        nna->max = WQ_DFL_MIN_ACTIVE;
        atomic_set(&nna->nr, 0);
        raw_spin_lock_init(&nna->lock);
        INIT_LIST_HEAD(&nna->pending_pwqs);
}

/*
 * Each node's nr_active counter will be accessed mostly from its own node and
 * should be allocated in the node.
 */
static int alloc_node_nr_active(struct wq_node_nr_active **nna_ar)
{
        struct wq_node_nr_active *nna;
        int node;

        for_each_node(node) {
                nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);
                if (!nna)
                        goto err_free;
                init_node_nr_active(nna);
                nna_ar[node] = nna;
        }

        /* [nr_node_ids] is used as the fallback */
        nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);
        if (!nna)
                goto err_free;
        init_node_nr_active(nna);
        nna_ar[nr_node_ids] = nna;

        return 0;

err_free:
        free_node_nr_active(nna_ar);
        return -ENOMEM;
}

static void rcu_free_wq(struct rcu_head *rcu)
{
        struct workqueue_struct *wq =
                container_of(rcu, struct workqueue_struct, rcu);

        if (wq->flags & WQ_UNBOUND)
                free_node_nr_active(wq->node_nr_active);

        wq_free_lockdep(wq);
        free_percpu(wq->cpu_pwq);
        free_workqueue_attrs(wq->unbound_attrs);
        kfree(wq);
}

static void rcu_free_pool(struct rcu_head *rcu)
{
        struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);

        ida_destroy(&pool->worker_ida);
        free_workqueue_attrs(pool->attrs);
        kfree(pool);
}

/**
 * put_unbound_pool - put a worker_pool
 * @pool: worker_pool to put
 *
 * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU
 * safe manner.  get_unbound_pool() calls this function on its failure path
 * and this function should be able to release pools which went through,
 * successfully or not, init_worker_pool().
 *
 * Should be called with wq_pool_mutex held.
 */
static void put_unbound_pool(struct worker_pool *pool)
{
        struct worker *worker;
        LIST_HEAD(cull_list);

        lockdep_assert_held(&wq_pool_mutex);

        if (--pool->refcnt)
                return;

        /* sanity checks */
        if (WARN_ON(!(pool->cpu < 0)) ||
            WARN_ON(!list_empty(&pool->worklist)))
                return;

        /* release id and unhash */
        if (pool->id >= 0)
                idr_remove(&worker_pool_idr, pool->id);
        hash_del(&pool->hash_node);

        /*
         * Become the manager and destroy all workers.  This prevents
         * @pool's workers from blocking on attach_mutex.  We're the last
         * manager and @pool gets freed with the flag set.
         *
         * Having a concurrent manager is quite unlikely to happen as we can
         * only get here with
         *   pwq->refcnt == pool->refcnt == 0
         * which implies no work queued to the pool, which implies no worker can
         * become the manager. However a worker could have taken the role of
         * manager before the refcnts dropped to 0, since maybe_create_worker()
         * drops pool->lock
         */
        while (true) {
                rcuwait_wait_event(&manager_wait,
                                   !(pool->flags & POOL_MANAGER_ACTIVE),
                                   TASK_UNINTERRUPTIBLE);

                mutex_lock(&wq_pool_attach_mutex);
                raw_spin_lock_irq(&pool->lock);
                if (!(pool->flags & POOL_MANAGER_ACTIVE)) {
                        pool->flags |= POOL_MANAGER_ACTIVE;
                        break;
                }
                raw_spin_unlock_irq(&pool->lock);
                mutex_unlock(&wq_pool_attach_mutex);
        }

        while ((worker = first_idle_worker(pool)))
                set_worker_dying(worker, &cull_list);
        WARN_ON(pool->nr_workers || pool->nr_idle);
        raw_spin_unlock_irq(&pool->lock);

        detach_dying_workers(&cull_list);

        mutex_unlock(&wq_pool_attach_mutex);

        reap_dying_workers(&cull_list);

        /* shut down the timers */
        timer_delete_sync(&pool->idle_timer);
        cancel_work_sync(&pool->idle_cull_work);
        timer_delete_sync(&pool->mayday_timer);

        /* RCU protected to allow dereferences from get_work_pool() */
        call_rcu(&pool->rcu, rcu_free_pool);
}

/**
 * get_unbound_pool - get a worker_pool with the specified attributes
 * @attrs: the attributes of the worker_pool to get
 *
 * Obtain a worker_pool which has the same attributes as @attrs, bump the
 * reference count and return it.  If there already is a matching
 * worker_pool, it will be used; otherwise, this function attempts to
 * create a new one.
 *
 * Should be called with wq_pool_mutex held.
 *
 * Return: On success, a worker_pool with the same attributes as @attrs.
 * On failure, %NULL.
 */
static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
{
        struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];
        u32 hash = wqattrs_hash(attrs);
        struct worker_pool *pool;
        int pod, node = NUMA_NO_NODE;

        lockdep_assert_held(&wq_pool_mutex);

        /* do we already have a matching pool? */
        hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
                if (wqattrs_equal(pool->attrs, attrs)) {
                        pool->refcnt++;
                        return pool;
                }
        }

        /* If __pod_cpumask is contained inside a NUMA pod, that's our node */
        for (pod = 0; pod < pt->nr_pods; pod++) {
                if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {
                        node = pt->pod_node[pod];
                        break;
                }
        }

        /* nope, create a new one */
        pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);
        if (!pool || init_worker_pool(pool) < 0)
                goto fail;

        pool->node = node;
        copy_workqueue_attrs(pool->attrs, attrs);
        wqattrs_clear_for_pool(pool->attrs);

        if (worker_pool_assign_id(pool) < 0)
                goto fail;

        /* create and start the initial worker */
        if (wq_online && !create_worker(pool))
                goto fail;

        /* install */
        hash_add(unbound_pool_hash, &pool->hash_node, hash);

        return pool;
fail:
        if (pool)
                put_unbound_pool(pool);
        return NULL;
}

/*
 * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero
 * refcnt and needs to be destroyed.
 */
static void pwq_release_workfn(struct kthread_work *work)
{
        struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
                                                  release_work);
        struct workqueue_struct *wq = pwq->wq;
        struct worker_pool *pool = pwq->pool;
        bool is_last = false;

        /*
         * When @pwq is not linked, it doesn't hold any reference to the
         * @wq, and @wq is invalid to access.
         */
        if (!list_empty(&pwq->pwqs_node)) {
                mutex_lock(&wq->mutex);
                list_del_rcu(&pwq->pwqs_node);
                is_last = list_empty(&wq->pwqs);

                /*
                 * For ordered workqueue with a plugged dfl_pwq, restart it now.
                 */
                if (!is_last && (wq->flags & __WQ_ORDERED))
                        unplug_oldest_pwq(wq);

                mutex_unlock(&wq->mutex);
        }

        if (wq->flags & WQ_UNBOUND) {
                mutex_lock(&wq_pool_mutex);
                put_unbound_pool(pool);
                mutex_unlock(&wq_pool_mutex);
        }

        if (!list_empty(&pwq->pending_node)) {
                struct wq_node_nr_active *nna =
                        wq_node_nr_active(pwq->wq, pwq->pool->node);

                raw_spin_lock_irq(&nna->lock);
                list_del_init(&pwq->pending_node);
                raw_spin_unlock_irq(&nna->lock);
        }

        kfree_rcu(pwq, rcu);

        /*
         * If we're the last pwq going away, @wq is already dead and no one
         * is gonna access it anymore.  Schedule RCU free.
         */
        if (is_last) {
                wq_unregister_lockdep(wq);
                call_rcu(&wq->rcu, rcu_free_wq);
        }
}

/* initialize newly allocated @pwq which is associated with @wq and @pool */
static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
                     struct worker_pool *pool)
{
        BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK);

        memset(pwq, 0, sizeof(*pwq));

        pwq->pool = pool;
        pwq->wq = wq;
        pwq->flush_color = -1;
        pwq->refcnt = 1;
        INIT_LIST_HEAD(&pwq->inactive_works);
        INIT_LIST_HEAD(&pwq->pending_node);
        INIT_LIST_HEAD(&pwq->pwqs_node);
        INIT_LIST_HEAD(&pwq->mayday_node);
        kthread_init_work(&pwq->release_work, pwq_release_workfn);
}

/* sync @pwq with the current state of its associated wq and link it */
static void link_pwq(struct pool_workqueue *pwq)
{
        struct workqueue_struct *wq = pwq->wq;

        lockdep_assert_held(&wq->mutex);

        /* may be called multiple times, ignore if already linked */
        if (!list_empty(&pwq->pwqs_node))
                return;

        /* set the matching work_color */
        pwq->work_color = wq->work_color;

        /* link in @pwq */
        list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);
}

/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
                                        const struct workqueue_attrs *attrs)
{
        struct worker_pool *pool;
        struct pool_workqueue *pwq;

        lockdep_assert_held(&wq_pool_mutex);

        pool = get_unbound_pool(attrs);
        if (!pool)
                return NULL;

        pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
        if (!pwq) {
                put_unbound_pool(pool);
                return NULL;
        }

        init_pwq(pwq, wq, pool);
        return pwq;
}

static void apply_wqattrs_lock(void)
{
        mutex_lock(&wq_pool_mutex);
}

static void apply_wqattrs_unlock(void)
{
        mutex_unlock(&wq_pool_mutex);
}

/**
 * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
 * @attrs: the wq_attrs of the default pwq of the target workqueue
 * @cpu: the target CPU
 *
 * Calculate the cpumask a workqueue with @attrs should use on @pod.
 * The result is stored in @attrs->__pod_cpumask.
 *
 * If pod affinity is not enabled, @attrs->cpumask is always used. If enabled
 * and @pod has online CPUs requested by @attrs, the returned cpumask is the
 * intersection of the possible CPUs of @pod and @attrs->cpumask.
 *
 * The caller is responsible for ensuring that the cpumask of @pod stays stable.
 */
static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu)
{
        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
        int pod = pt->cpu_pod[cpu];

        /* calculate possible CPUs in @pod that @attrs wants */
        cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);
        /* does @pod have any online CPUs @attrs wants? */
        if (!cpumask_intersects(attrs->__pod_cpumask, wq_online_cpumask)) {
                cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);
                return;
        }
}

/* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */
static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,
                                        int cpu, struct pool_workqueue *pwq)
{
        struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);
        struct pool_workqueue *old_pwq;

        lockdep_assert_held(&wq_pool_mutex);
        lockdep_assert_held(&wq->mutex);

        /* link_pwq() can handle duplicate calls */
        link_pwq(pwq);

        old_pwq = rcu_access_pointer(*slot);
        rcu_assign_pointer(*slot, pwq);
        return old_pwq;
}

/* context to store the prepared attrs & pwqs before applying */
struct apply_wqattrs_ctx {
        struct workqueue_struct        *wq;                /* target workqueue */
        struct workqueue_attrs        *attrs;                /* attrs to apply */
        struct list_head        list;                /* queued for batching commit */
        struct pool_workqueue        *dfl_pwq;
        struct pool_workqueue        *pwq_tbl[];
};

/* free the resources after success or abort */
static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
{
        if (ctx) {
                int cpu;

                for_each_possible_cpu(cpu)
                        put_pwq_unlocked(ctx->pwq_tbl[cpu]);
                put_pwq_unlocked(ctx->dfl_pwq);

                free_workqueue_attrs(ctx->attrs);

                kfree(ctx);
        }
}

/* allocate the attrs and pwqs for later installation */
static struct apply_wqattrs_ctx *
apply_wqattrs_prepare(struct workqueue_struct *wq,
                      const struct workqueue_attrs *attrs,
                      const cpumask_var_t unbound_cpumask)
{
        struct apply_wqattrs_ctx *ctx;
        struct workqueue_attrs *new_attrs;
        int cpu;

        lockdep_assert_held(&wq_pool_mutex);

        if (WARN_ON(attrs->affn_scope < 0 ||
                    attrs->affn_scope >= WQ_AFFN_NR_TYPES))
                return ERR_PTR(-EINVAL);

        ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_cpu_ids), GFP_KERNEL);

        new_attrs = alloc_workqueue_attrs();
        if (!ctx || !new_attrs)
                goto out_free;

        /*
         * If something goes wrong during CPU up/down, we'll fall back to
         * the default pwq covering whole @attrs->cpumask.  Always create
         * it even if we don't use it immediately.
         */
        copy_workqueue_attrs(new_attrs, attrs);
        wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
        cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
        ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
        if (!ctx->dfl_pwq)
                goto out_free;

        for_each_possible_cpu(cpu) {
                if (new_attrs->ordered) {
                        ctx->dfl_pwq->refcnt++;
                        ctx->pwq_tbl[cpu] = ctx->dfl_pwq;
                } else {
                        wq_calc_pod_cpumask(new_attrs, cpu);
                        ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);
                        if (!ctx->pwq_tbl[cpu])
                                goto out_free;
                }
        }

        /* save the user configured attrs and sanitize it. */
        copy_workqueue_attrs(new_attrs, attrs);
        cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
        cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
        ctx->attrs = new_attrs;

        /*
         * For initialized ordered workqueues, there should only be one pwq
         * (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution
         * of newly queued work items until execution of older work items in
         * the old pwq's have completed.
         */
        if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))
                ctx->dfl_pwq->plugged = true;

        ctx->wq = wq;
        return ctx;

out_free:
        free_workqueue_attrs(new_attrs);
        apply_wqattrs_cleanup(ctx);
        return ERR_PTR(-ENOMEM);
}

/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
{
        int cpu;

        /* all pwqs have been created successfully, let's install'em */
        mutex_lock(&ctx->wq->mutex);

        copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);

        /* save the previous pwqs and install the new ones */
        for_each_possible_cpu(cpu)
                ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,
                                                        ctx->pwq_tbl[cpu]);
        ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq);

        /* update node_nr_active->max */
        wq_update_node_max_active(ctx->wq, -1);

        /* rescuer needs to respect wq cpumask changes */
        if (ctx->wq->rescuer)
                set_cpus_allowed_ptr(ctx->wq->rescuer->task,
                                     unbound_effective_cpumask(ctx->wq));

        mutex_unlock(&ctx->wq->mutex);
}

static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
                                        const struct workqueue_attrs *attrs)
{
        struct apply_wqattrs_ctx *ctx;

        /* only unbound workqueues can change attributes */
        if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
                return -EINVAL;

        ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
        if (IS_ERR(ctx))
                return PTR_ERR(ctx);

        /* the ctx has been prepared successfully, let's commit it */
        apply_wqattrs_commit(ctx);
        apply_wqattrs_cleanup(ctx);

        return 0;
}

/**
 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
 * @wq: the target workqueue
 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
 *
 * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps
 * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that
 * work items are affine to the pod it was issued on. Older pwqs are released as
 * in-flight work items finish. Note that a work item which repeatedly requeues
 * itself back-to-back will stay on its current pwq.
 *
 * Performs GFP_KERNEL allocations.
 *
 * Return: 0 on success and -errno on failure.
 */
int apply_workqueue_attrs(struct workqueue_struct *wq,
                          const struct workqueue_attrs *attrs)
{
        int ret;

        mutex_lock(&wq_pool_mutex);
        ret = apply_workqueue_attrs_locked(wq, attrs);
        mutex_unlock(&wq_pool_mutex);

        return ret;
}

/**
 * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug
 * @wq: the target workqueue
 * @cpu: the CPU to update the pwq slot for
 *
 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
 * %CPU_DOWN_FAILED.  @cpu is in the same pod of the CPU being hot[un]plugged.
 *
 *
 * If pod affinity can't be adjusted due to memory allocation failure, it falls
 * back to @wq->dfl_pwq which may not be optimal but is always correct.
 *
 * Note that when the last allowed CPU of a pod goes offline for a workqueue
 * with a cpumask spanning multiple pods, the workers which were already
 * executing the work items for the workqueue will lose their CPU affinity and
 * may execute on any CPU. This is similar to how per-cpu workqueues behave on
 * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's
 * responsibility to flush the work item from CPU_DOWN_PREPARE.
 */
static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
{
        struct pool_workqueue *old_pwq = NULL, *pwq;
        struct workqueue_attrs *target_attrs;

        lockdep_assert_held(&wq_pool_mutex);

        if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
                return;

        /*
         * We don't wanna alloc/free wq_attrs for each wq for each CPU.
         * Let's use a preallocated one.  The following buf is protected by
         * CPU hotplug exclusion.
         */
        target_attrs = unbound_wq_update_pwq_attrs_buf;

        copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
        wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask);

        /* nothing to do if the target cpumask matches the current pwq */
        wq_calc_pod_cpumask(target_attrs, cpu);
        if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))
                return;

        /* create a new pwq */
        pwq = alloc_unbound_pwq(wq, target_attrs);
        if (!pwq) {
                pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",
                        wq->name);
                goto use_dfl_pwq;
        }

        /* Install the new pwq. */
        mutex_lock(&wq->mutex);
        old_pwq = install_unbound_pwq(wq, cpu, pwq);
        goto out_unlock;

use_dfl_pwq:
        mutex_lock(&wq->mutex);
        pwq = unbound_pwq(wq, -1);
        raw_spin_lock_irq(&pwq->pool->lock);
        get_pwq(pwq);
        raw_spin_unlock_irq(&pwq->pool->lock);
        old_pwq = install_unbound_pwq(wq, cpu, pwq);
out_unlock:
        mutex_unlock(&wq->mutex);
        put_pwq_unlocked(old_pwq);
}

static int alloc_and_link_pwqs(struct workqueue_struct *wq)
{
        bool highpri = wq->flags & WQ_HIGHPRI;
        int cpu, ret;

        lockdep_assert_held(&wq_pool_mutex);

        wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);
        if (!wq->cpu_pwq)
                goto enomem;

        if (!(wq->flags & WQ_UNBOUND)) {
                struct worker_pool __percpu *pools;

                if (wq->flags & WQ_BH)
                        pools = bh_worker_pools;
                else
                        pools = cpu_worker_pools;

                for_each_possible_cpu(cpu) {
                        struct pool_workqueue **pwq_p;
                        struct worker_pool *pool;

                        pool = &(per_cpu_ptr(pools, cpu)[highpri]);
                        pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu);

                        *pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,
                                                       pool->node);
                        if (!*pwq_p)
                                goto enomem;

                        init_pwq(*pwq_p, wq, pool);

                        mutex_lock(&wq->mutex);
                        link_pwq(*pwq_p);
                        mutex_unlock(&wq->mutex);
                }
                return 0;
        }

        if (wq->flags & __WQ_ORDERED) {
                struct pool_workqueue *dfl_pwq;

                ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);
                /* there should only be single pwq for ordering guarantee */
                dfl_pwq = rcu_access_pointer(wq->dfl_pwq);
                WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||
                              wq->pwqs.prev != &dfl_pwq->pwqs_node),
                     "ordering guarantee broken for workqueue %s\n", wq->name);
        } else {
                ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);
        }

        return ret;

enomem:
        if (wq->cpu_pwq) {
                for_each_possible_cpu(cpu) {
                        struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);

                        if (pwq)
                                kmem_cache_free(pwq_cache, pwq);
                }
                free_percpu(wq->cpu_pwq);
                wq->cpu_pwq = NULL;
        }
        return -ENOMEM;
}

static int wq_clamp_max_active(int max_active, unsigned int flags,
                               const char *name)
{
        if (max_active < 1 || max_active > WQ_MAX_ACTIVE)
                pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
                        max_active, name, 1, WQ_MAX_ACTIVE);

        return clamp_val(max_active, 1, WQ_MAX_ACTIVE);
}

/*
 * Workqueues which may be used during memory reclaim should have a rescuer
 * to guarantee forward progress.
 */
static int init_rescuer(struct workqueue_struct *wq)
{
        struct worker *rescuer;
        char id_buf[WORKER_ID_LEN];
        int ret;

        lockdep_assert_held(&wq_pool_mutex);

        if (!(wq->flags & WQ_MEM_RECLAIM))
                return 0;

        rescuer = alloc_worker(NUMA_NO_NODE);
        if (!rescuer) {
                pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",
                       wq->name);
                return -ENOMEM;
        }

        rescuer->rescue_wq = wq;
        format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL);

        rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);
        if (IS_ERR(rescuer->task)) {
                ret = PTR_ERR(rescuer->task);
                pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",
                       wq->name, ERR_PTR(ret));
                kfree(rescuer);
                return ret;
        }

        wq->rescuer = rescuer;
        if (wq->flags & WQ_UNBOUND)
                kthread_bind_mask(rescuer->task, unbound_effective_cpumask(wq));
        else
                kthread_bind_mask(rescuer->task, cpu_possible_mask);
        wake_up_process(rescuer->task);

        return 0;
}

/**
 * wq_adjust_max_active - update a wq's max_active to the current setting
 * @wq: target workqueue
 *
 * If @wq isn't freezing, set @wq->max_active to the saved_max_active and
 * activate inactive work items accordingly. If @wq is freezing, clear
 * @wq->max_active to zero.
 */
static void wq_adjust_max_active(struct workqueue_struct *wq)
{
        bool activated;
        int new_max, new_min;

        lockdep_assert_held(&wq->mutex);

        if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {
                new_max = 0;
                new_min = 0;
        } else {
                new_max = wq->saved_max_active;
                new_min = wq->saved_min_active;
        }

        if (wq->max_active == new_max && wq->min_active == new_min)
                return;

        /*
         * Update @wq->max/min_active and then kick inactive work items if more
         * active work items are allowed. This doesn't break work item ordering
         * because new work items are always queued behind existing inactive
         * work items if there are any.
         */
        WRITE_ONCE(wq->max_active, new_max);
        WRITE_ONCE(wq->min_active, new_min);

        if (wq->flags & WQ_UNBOUND)
                wq_update_node_max_active(wq, -1);

        if (new_max == 0)
                return;

        /*
         * Round-robin through pwq's activating the first inactive work item
         * until max_active is filled.
         */
        do {
                struct pool_workqueue *pwq;

                activated = false;
                for_each_pwq(pwq, wq) {
                        unsigned long irq_flags;

                        /* can be called during early boot w/ irq disabled */
                        raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
                        if (pwq_activate_first_inactive(pwq, true)) {
                                activated = true;
                                kick_pool(pwq->pool);
                        }
                        raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
                }
        } while (activated);
}

__printf(1, 0)
static struct workqueue_struct *__alloc_workqueue(const char *fmt,
                                                  unsigned int flags,
                                                  int max_active, va_list args)
{
        struct workqueue_struct *wq;
        size_t wq_size;
        int name_len;

        if (flags & WQ_BH) {
                if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))
                        return NULL;
                if (WARN_ON_ONCE(max_active))
                        return NULL;
        }

        /* see the comment above the definition of WQ_POWER_EFFICIENT */
        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
                flags |= WQ_UNBOUND;

        /* allocate wq and format name */
        if (flags & WQ_UNBOUND)
                wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);
        else
                wq_size = sizeof(*wq);

        wq = kzalloc(wq_size, GFP_KERNEL);
        if (!wq)
                return NULL;

        if (flags & WQ_UNBOUND) {
                wq->unbound_attrs = alloc_workqueue_attrs();
                if (!wq->unbound_attrs)
                        goto err_free_wq;
        }

        name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args);

        if (name_len >= WQ_NAME_LEN)
                pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",
                             wq->name);

        if (flags & WQ_BH) {
                /*
                 * BH workqueues always share a single execution context per CPU
                 * and don't impose any max_active limit.
                 */
                max_active = INT_MAX;
        } else {
                max_active = max_active ?: WQ_DFL_ACTIVE;
                max_active = wq_clamp_max_active(max_active, flags, wq->name);
        }

        /* init wq */
        wq->flags = flags;
        wq->max_active = max_active;
        wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);
        wq->saved_max_active = wq->max_active;
        wq->saved_min_active = wq->min_active;
        mutex_init(&wq->mutex);
        atomic_set(&wq->nr_pwqs_to_flush, 0);
        INIT_LIST_HEAD(&wq->pwqs);
        INIT_LIST_HEAD(&wq->flusher_queue);
        INIT_LIST_HEAD(&wq->flusher_overflow);
        INIT_LIST_HEAD(&wq->maydays);

        INIT_LIST_HEAD(&wq->list);

        if (flags & WQ_UNBOUND) {
                if (alloc_node_nr_active(wq->node_nr_active) < 0)
                        goto err_free_wq;
        }

        /*
         * wq_pool_mutex protects the workqueues list, allocations of PWQs,
         * and the global freeze state.
         */
        apply_wqattrs_lock();

        if (alloc_and_link_pwqs(wq) < 0)
                goto err_unlock_free_node_nr_active;

        mutex_lock(&wq->mutex);
        wq_adjust_max_active(wq);
        mutex_unlock(&wq->mutex);

        list_add_tail_rcu(&wq->list, &workqueues);

        if (wq_online && init_rescuer(wq) < 0)
                goto err_unlock_destroy;

        apply_wqattrs_unlock();

        if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
                goto err_destroy;

        return wq;

err_unlock_free_node_nr_active:
        apply_wqattrs_unlock();
        /*
         * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,
         * flushing the pwq_release_worker ensures that the pwq_release_workfn()
         * completes before calling kfree(wq).
         */
        if (wq->flags & WQ_UNBOUND) {
                kthread_flush_worker(pwq_release_worker);
                free_node_nr_active(wq->node_nr_active);
        }
err_free_wq:
        free_workqueue_attrs(wq->unbound_attrs);
        kfree(wq);
        return NULL;
err_unlock_destroy:
        apply_wqattrs_unlock();
err_destroy:
        destroy_workqueue(wq);
        return NULL;
}

__printf(1, 4)
struct workqueue_struct *alloc_workqueue(const char *fmt,
                                         unsigned int flags,
                                         int max_active, ...)
{
        struct workqueue_struct *wq;
        va_list args;

        va_start(args, max_active);
        wq = __alloc_workqueue(fmt, flags, max_active, args);
        va_end(args);
        if (!wq)
                return NULL;

        wq_init_lockdep(wq);

        return wq;
}
EXPORT_SYMBOL_GPL(alloc_workqueue);

#ifdef CONFIG_LOCKDEP
__printf(1, 5)
struct workqueue_struct *
alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags,
                            int max_active, struct lockdep_map *lockdep_map, ...)
{
        struct workqueue_struct *wq;
        va_list args;

        va_start(args, lockdep_map);
        wq = __alloc_workqueue(fmt, flags, max_active, args);
        va_end(args);
        if (!wq)
                return NULL;

        wq->lockdep_map = lockdep_map;

        return wq;
}
EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);
#endif

static bool pwq_busy(struct pool_workqueue *pwq)
{
        int i;

        for (i = 0; i < WORK_NR_COLORS; i++)
                if (pwq->nr_in_flight[i])
                        return true;

        if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))
                return true;
        if (!pwq_is_empty(pwq))
                return true;

        return false;
}

/**
 * destroy_workqueue - safely terminate a workqueue
 * @wq: target workqueue
 *
 * Safely destroy a workqueue. All work currently pending will be done first.
 *
 * This function does NOT guarantee that non-pending work that has been
 * submitted with queue_delayed_work() and similar functions will be done
 * before destroying the workqueue. The fundamental problem is that, currently,
 * the workqueue has no way of accessing non-pending delayed_work. delayed_work
 * is only linked on the timer-side. All delayed_work must, therefore, be
 * canceled before calling this function.
 *
 * TODO: It would be better if the problem described above wouldn't exist and
 * destroy_workqueue() would cleanly cancel all pending and non-pending
 * delayed_work.
 */
void destroy_workqueue(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        int cpu;

        /*
         * Remove it from sysfs first so that sanity check failure doesn't
         * lead to sysfs name conflicts.
         */
        workqueue_sysfs_unregister(wq);

        /* mark the workqueue destruction is in progress */
        mutex_lock(&wq->mutex);
        wq->flags |= __WQ_DESTROYING;
        mutex_unlock(&wq->mutex);

        /* drain it before proceeding with destruction */
        drain_workqueue(wq);

        /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
        if (wq->rescuer) {
                struct worker *rescuer = wq->rescuer;

                /* this prevents new queueing */
                raw_spin_lock_irq(&wq_mayday_lock);
                wq->rescuer = NULL;
                raw_spin_unlock_irq(&wq_mayday_lock);

                /* rescuer will empty maydays list before exiting */
                kthread_stop(rescuer->task);
                kfree(rescuer);
        }

        /*
         * Sanity checks - grab all the locks so that we wait for all
         * in-flight operations which may do put_pwq().
         */
        mutex_lock(&wq_pool_mutex);
        mutex_lock(&wq->mutex);
        for_each_pwq(pwq, wq) {
                raw_spin_lock_irq(&pwq->pool->lock);
                if (WARN_ON(pwq_busy(pwq))) {
                        pr_warn("%s: %s has the following busy pwq\n",
                                __func__, wq->name);
                        show_pwq(pwq);
                        raw_spin_unlock_irq(&pwq->pool->lock);
                        mutex_unlock(&wq->mutex);
                        mutex_unlock(&wq_pool_mutex);
                        show_one_workqueue(wq);
                        return;
                }
                raw_spin_unlock_irq(&pwq->pool->lock);
        }
        mutex_unlock(&wq->mutex);

        /*
         * wq list is used to freeze wq, remove from list after
         * flushing is complete in case freeze races us.
         */
        list_del_rcu(&wq->list);
        mutex_unlock(&wq_pool_mutex);

        /*
         * We're the sole accessor of @wq. Directly access cpu_pwq and dfl_pwq
         * to put the base refs. @wq will be auto-destroyed from the last
         * pwq_put. RCU read lock prevents @wq from going away from under us.
         */
        rcu_read_lock();

        for_each_possible_cpu(cpu) {
                put_pwq_unlocked(unbound_pwq(wq, cpu));
                RCU_INIT_POINTER(*unbound_pwq_slot(wq, cpu), NULL);
        }

        put_pwq_unlocked(unbound_pwq(wq, -1));
        RCU_INIT_POINTER(*unbound_pwq_slot(wq, -1), NULL);

        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(destroy_workqueue);

/**
 * workqueue_set_max_active - adjust max_active of a workqueue
 * @wq: target workqueue
 * @max_active: new max_active value.
 *
 * Set max_active of @wq to @max_active. See the alloc_workqueue() function
 * comment.
 *
 * CONTEXT:
 * Don't call from IRQ context.
 */
void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
{
        /* max_active doesn't mean anything for BH workqueues */
        if (WARN_ON(wq->flags & WQ_BH))
                return;
        /* disallow meddling with max_active for ordered workqueues */
        if (WARN_ON(wq->flags & __WQ_ORDERED))
                return;

        max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);

        mutex_lock(&wq->mutex);

        wq->saved_max_active = max_active;
        if (wq->flags & WQ_UNBOUND)
                wq->saved_min_active = min(wq->saved_min_active, max_active);

        wq_adjust_max_active(wq);

        mutex_unlock(&wq->mutex);
}
EXPORT_SYMBOL_GPL(workqueue_set_max_active);

/**
 * workqueue_set_min_active - adjust min_active of an unbound workqueue
 * @wq: target unbound workqueue
 * @min_active: new min_active value
 *
 * Set min_active of an unbound workqueue. Unlike other types of workqueues, an
 * unbound workqueue is not guaranteed to be able to process max_active
 * interdependent work items. Instead, an unbound workqueue is guaranteed to be
 * able to process min_active number of interdependent work items which is
 * %WQ_DFL_MIN_ACTIVE by default.
 *
 * Use this function to adjust the min_active value between 0 and the current
 * max_active.
 */
void workqueue_set_min_active(struct workqueue_struct *wq, int min_active)
{
        /* min_active is only meaningful for non-ordered unbound workqueues */
        if (WARN_ON((wq->flags & (WQ_BH | WQ_UNBOUND | __WQ_ORDERED)) !=
                    WQ_UNBOUND))
                return;

        mutex_lock(&wq->mutex);
        wq->saved_min_active = clamp(min_active, 0, wq->saved_max_active);
        wq_adjust_max_active(wq);
        mutex_unlock(&wq->mutex);
}

/**
 * current_work - retrieve %current task's work struct
 *
 * Determine if %current task is a workqueue worker and what it's working on.
 * Useful to find out the context that the %current task is running in.
 *
 * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
 */
struct work_struct *current_work(void)
{
        struct worker *worker = current_wq_worker();

        return worker ? worker->current_work : NULL;
}
EXPORT_SYMBOL(current_work);

/**
 * current_is_workqueue_rescuer - is %current workqueue rescuer?
 *
 * Determine whether %current is a workqueue rescuer.  Can be used from
 * work functions to determine whether it's being run off the rescuer task.
 *
 * Return: %true if %current is a workqueue rescuer. %false otherwise.
 */
bool current_is_workqueue_rescuer(void)
{
        struct worker *worker = current_wq_worker();

        return worker && worker->rescue_wq;
}

/**
 * workqueue_congested - test whether a workqueue is congested
 * @cpu: CPU in question
 * @wq: target workqueue
 *
 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
 * no synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 *
 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
 *
 * With the exception of ordered workqueues, all workqueues have per-cpu
 * pool_workqueues, each with its own congested state. A workqueue being
 * congested on one CPU doesn't mean that the workqueue is contested on any
 * other CPUs.
 *
 * Return:
 * %true if congested, %false otherwise.
 */
bool workqueue_congested(int cpu, struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        bool ret;

        rcu_read_lock();
        preempt_disable();

        if (cpu == WORK_CPU_UNBOUND)
                cpu = smp_processor_id();

        pwq = *per_cpu_ptr(wq->cpu_pwq, cpu);
        ret = !list_empty(&pwq->inactive_works);

        preempt_enable();
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(workqueue_congested);

/**
 * work_busy - test whether a work is currently pending or running
 * @work: the work to be tested
 *
 * Test whether @work is currently pending or running.  There is no
 * synchronization around this function and the test result is
 * unreliable and only useful as advisory hints or for debugging.
 *
 * Return:
 * OR'd bitmask of WORK_BUSY_* bits.
 */
unsigned int work_busy(struct work_struct *work)
{
        struct worker_pool *pool;
        unsigned long irq_flags;
        unsigned int ret = 0;

        if (work_pending(work))
                ret |= WORK_BUSY_PENDING;

        rcu_read_lock();
        pool = get_work_pool(work);
        if (pool) {
                raw_spin_lock_irqsave(&pool->lock, irq_flags);
                if (find_worker_executing_work(pool, work))
                        ret |= WORK_BUSY_RUNNING;
                raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
        }
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL_GPL(work_busy);

/**
 * set_worker_desc - set description for the current work item
 * @fmt: printf-style format string
 * @...: arguments for the format string
 *
 * This function can be called by a running work function to describe what
 * the work item is about.  If the worker task gets dumped, this
 * information will be printed out together to help debugging.  The
 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
 */
void set_worker_desc(const char *fmt, ...)
{
        struct worker *worker = current_wq_worker();
        va_list args;

        if (worker) {
                va_start(args, fmt);
                vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
                va_end(args);
        }
}
EXPORT_SYMBOL_GPL(set_worker_desc);

/**
 * print_worker_info - print out worker information and description
 * @log_lvl: the log level to use when printing
 * @task: target task
 *
 * If @task is a worker and currently executing a work item, print out the
 * name of the workqueue being serviced and worker description set with
 * set_worker_desc() by the currently executing work item.
 *
 * This function can be safely called on any task as long as the
 * task_struct itself is accessible.  While safe, this function isn't
 * synchronized and may print out mixups or garbages of limited length.
 */
void print_worker_info(const char *log_lvl, struct task_struct *task)
{
        work_func_t *fn = NULL;
        char name[WQ_NAME_LEN] = { };
        char desc[WORKER_DESC_LEN] = { };
        struct pool_workqueue *pwq = NULL;
        struct workqueue_struct *wq = NULL;
        struct worker *worker;

        if (!(task->flags & PF_WQ_WORKER))
                return;

        /*
         * This function is called without any synchronization and @task
         * could be in any state.  Be careful with dereferences.
         */
        worker = kthread_probe_data(task);

        /*
         * Carefully copy the associated workqueue's workfn, name and desc.
         * Keep the original last '\0' in case the original is garbage.
         */
        copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
        copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
        copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
        copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
        copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);

        if (fn || name[0] || desc[0]) {
                printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
                if (strcmp(name, desc))
                        pr_cont(" (%s)", desc);
                pr_cont("\n");
        }
}

static void pr_cont_pool_info(struct worker_pool *pool)
{
        pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
        if (pool->node != NUMA_NO_NODE)
                pr_cont(" node=%d", pool->node);
        pr_cont(" flags=0x%x", pool->flags);
        if (pool->flags & POOL_BH)
                pr_cont(" bh%s",
                        pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
        else
                pr_cont(" nice=%d", pool->attrs->nice);
}

static void pr_cont_worker_id(struct worker *worker)
{
        struct worker_pool *pool = worker->pool;

        if (pool->flags & WQ_BH)
                pr_cont("bh%s",
                        pool->attrs->nice == HIGHPRI_NICE_LEVEL ? "-hi" : "");
        else
                pr_cont("%d%s", task_pid_nr(worker->task),
                        worker->rescue_wq ? "(RESCUER)" : "");
}

struct pr_cont_work_struct {
        bool comma;
        work_func_t func;
        long ctr;
};

static void pr_cont_work_flush(bool comma, work_func_t func, struct pr_cont_work_struct *pcwsp)
{
        if (!pcwsp->ctr)
                goto out_record;
        if (func == pcwsp->func) {
                pcwsp->ctr++;
                return;
        }
        if (pcwsp->ctr == 1)
                pr_cont("%s %ps", pcwsp->comma ? "," : "", pcwsp->func);
        else
                pr_cont("%s %ld*%ps", pcwsp->comma ? "," : "", pcwsp->ctr, pcwsp->func);
        pcwsp->ctr = 0;
out_record:
        if ((long)func == -1L)
                return;
        pcwsp->comma = comma;
        pcwsp->func = func;
        pcwsp->ctr = 1;
}

static void pr_cont_work(bool comma, struct work_struct *work, struct pr_cont_work_struct *pcwsp)
{
        if (work->func == wq_barrier_func) {
                struct wq_barrier *barr;

                barr = container_of(work, struct wq_barrier, work);

                pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
                pr_cont("%s BAR(%d)", comma ? "," : "",
                        task_pid_nr(barr->task));
        } else {
                if (!comma)
                        pr_cont_work_flush(comma, (work_func_t)-1, pcwsp);
                pr_cont_work_flush(comma, work->func, pcwsp);
        }
}

static void show_pwq(struct pool_workqueue *pwq)
{
        struct pr_cont_work_struct pcws = { .ctr = 0, };
        struct worker_pool *pool = pwq->pool;
        struct work_struct *work;
        struct worker *worker;
        bool has_in_flight = false, has_pending = false;
        int bkt;

        pr_info("  pwq %d:", pool->id);
        pr_cont_pool_info(pool);

        pr_cont(" active=%d refcnt=%d%s\n",
                pwq->nr_active, pwq->refcnt,
                !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");

        hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                if (worker->current_pwq == pwq) {
                        has_in_flight = true;
                        break;
                }
        }
        if (has_in_flight) {
                bool comma = false;

                pr_info("    in-flight:");
                hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                        if (worker->current_pwq != pwq)
                                continue;

                        pr_cont(" %s", comma ? "," : "");
                        pr_cont_worker_id(worker);
                        pr_cont(":%ps", worker->current_func);
                        list_for_each_entry(work, &worker->scheduled, entry)
                                pr_cont_work(false, work, &pcws);
                        pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                        comma = true;
                }
                pr_cont("\n");
        }

        list_for_each_entry(work, &pool->worklist, entry) {
                if (get_work_pwq(work) == pwq) {
                        has_pending = true;
                        break;
                }
        }
        if (has_pending) {
                bool comma = false;

                pr_info("    pending:");
                list_for_each_entry(work, &pool->worklist, entry) {
                        if (get_work_pwq(work) != pwq)
                                continue;

                        pr_cont_work(comma, work, &pcws);
                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
                }
                pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                pr_cont("\n");
        }

        if (!list_empty(&pwq->inactive_works)) {
                bool comma = false;

                pr_info("    inactive:");
                list_for_each_entry(work, &pwq->inactive_works, entry) {
                        pr_cont_work(comma, work, &pcws);
                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
                }
                pr_cont_work_flush(comma, (work_func_t)-1L, &pcws);
                pr_cont("\n");
        }
}

/**
 * show_one_workqueue - dump state of specified workqueue
 * @wq: workqueue whose state will be printed
 */
void show_one_workqueue(struct workqueue_struct *wq)
{
        struct pool_workqueue *pwq;
        bool idle = true;
        unsigned long irq_flags;

        for_each_pwq(pwq, wq) {
                if (!pwq_is_empty(pwq)) {
                        idle = false;
                        break;
                }
        }
        if (idle) /* Nothing to print for idle workqueue */
                return;

        pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);

        for_each_pwq(pwq, wq) {
                raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);
                if (!pwq_is_empty(pwq)) {
                        /*
                         * Defer printing to avoid deadlocks in console
                         * drivers that queue work while holding locks
                         * also taken in their write paths.
                         */
                        printk_deferred_enter();
                        show_pwq(pwq);
                        printk_deferred_exit();
                }
                raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);
                /*
                 * We could be printing a lot from atomic context, e.g.
                 * sysrq-t -> show_all_workqueues(). Avoid triggering
                 * hard lockup.
                 */
                touch_nmi_watchdog();
        }

}

/**
 * show_one_worker_pool - dump state of specified worker pool
 * @pool: worker pool whose state will be printed
 */
static void show_one_worker_pool(struct worker_pool *pool)
{
        struct worker *worker;
        bool first = true;
        unsigned long irq_flags;
        unsigned long hung = 0;

        raw_spin_lock_irqsave(&pool->lock, irq_flags);
        if (pool->nr_workers == pool->nr_idle)
                goto next_pool;

        /* How long the first pending work is waiting for a worker. */
        if (!list_empty(&pool->worklist))
                hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;

        /*
         * Defer printing to avoid deadlocks in console drivers that
         * queue work while holding locks also taken in their write
         * paths.
         */
        printk_deferred_enter();
        pr_info("pool %d:", pool->id);
        pr_cont_pool_info(pool);
        pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
        if (pool->manager)
                pr_cont(" manager: %d",
                        task_pid_nr(pool->manager->task));
        list_for_each_entry(worker, &pool->idle_list, entry) {
                pr_cont(" %s", first ? "idle: " : "");
                pr_cont_worker_id(worker);
                first = false;
        }
        pr_cont("\n");
        printk_deferred_exit();
next_pool:
        raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
        /*
         * We could be printing a lot from atomic context, e.g.
         * sysrq-t -> show_all_workqueues(). Avoid triggering
         * hard lockup.
         */
        touch_nmi_watchdog();

}

/**
 * show_all_workqueues - dump workqueue state
 *
 * Called from a sysrq handler and prints out all busy workqueues and pools.
 */
void show_all_workqueues(void)
{
        struct workqueue_struct *wq;
        struct worker_pool *pool;
        int pi;

        rcu_read_lock();

        pr_info("Showing busy workqueues and worker pools:\n");

        list_for_each_entry_rcu(wq, &workqueues, list)
                show_one_workqueue(wq);

        for_each_pool(pool, pi)
                show_one_worker_pool(pool);

        rcu_read_unlock();
}

/**
 * show_freezable_workqueues - dump freezable workqueue state
 *
 * Called from try_to_freeze_tasks() and prints out all freezable workqueues
 * still busy.
 */
void show_freezable_workqueues(void)
{
        struct workqueue_struct *wq;

        rcu_read_lock();

        pr_info("Showing freezable workqueues that are still busy:\n");

        list_for_each_entry_rcu(wq, &workqueues, list) {
                if (!(wq->flags & WQ_FREEZABLE))
                        continue;
                show_one_workqueue(wq);
        }

        rcu_read_unlock();
}

/* used to show worker information through /proc/PID/{comm,stat,status} */
void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
{
        /* stabilize PF_WQ_WORKER and worker pool association */
        mutex_lock(&wq_pool_attach_mutex);

        if (task->flags & PF_WQ_WORKER) {
                struct worker *worker = kthread_data(task);
                struct worker_pool *pool = worker->pool;
                int off;

                off = format_worker_id(buf, size, worker, pool);

                if (pool) {
                        raw_spin_lock_irq(&pool->lock);
                        /*
                         * ->desc tracks information (wq name or
                         * set_worker_desc()) for the latest execution.  If
                         * current, prepend '+', otherwise '-'.
                         */
                        if (worker->desc[0] != '\0') {
                                if (worker->current_work)
                                        scnprintf(buf + off, size - off, "+%s",
                                                  worker->desc);
                                else
                                        scnprintf(buf + off, size - off, "-%s",
                                                  worker->desc);
                        }
                        raw_spin_unlock_irq(&pool->lock);
                }
        } else {
                strscpy(buf, task->comm, size);
        }

        mutex_unlock(&wq_pool_attach_mutex);
}

#ifdef CONFIG_SMP

/*
 * CPU hotplug.
 *
 * There are two challenges in supporting CPU hotplug.  Firstly, there
 * are a lot of assumptions on strong associations among work, pwq and
 * pool which make migrating pending and scheduled works very
 * difficult to implement without impacting hot paths.  Secondly,
 * worker pools serve mix of short, long and very long running works making
 * blocked draining impractical.
 *
 * This is solved by allowing the pools to be disassociated from the CPU
 * running as an unbound one and allowing it to be reattached later if the
 * cpu comes back online.
 */

static void unbind_workers(int cpu)
{
        struct worker_pool *pool;
        struct worker *worker;

        for_each_cpu_worker_pool(pool, cpu) {
                mutex_lock(&wq_pool_attach_mutex);
                raw_spin_lock_irq(&pool->lock);

                /*
                 * We've blocked all attach/detach operations. Make all workers
                 * unbound and set DISASSOCIATED.  Before this, all workers
                 * must be on the cpu.  After this, they may become diasporas.
                 * And the preemption disabled section in their sched callbacks
                 * are guaranteed to see WORKER_UNBOUND since the code here
                 * is on the same cpu.
                 */
                for_each_pool_worker(worker, pool)
                        worker->flags |= WORKER_UNBOUND;

                pool->flags |= POOL_DISASSOCIATED;

                /*
                 * The handling of nr_running in sched callbacks are disabled
                 * now.  Zap nr_running.  After this, nr_running stays zero and
                 * need_more_worker() and keep_working() are always true as
                 * long as the worklist is not empty.  This pool now behaves as
                 * an unbound (in terms of concurrency management) pool which
                 * are served by workers tied to the pool.
                 */
                pool->nr_running = 0;

                /*
                 * With concurrency management just turned off, a busy
                 * worker blocking could lead to lengthy stalls.  Kick off
                 * unbound chain execution of currently pending work items.
                 */
                kick_pool(pool);

                raw_spin_unlock_irq(&pool->lock);

                for_each_pool_worker(worker, pool)
                        unbind_worker(worker);

                mutex_unlock(&wq_pool_attach_mutex);
        }
}

/**
 * rebind_workers - rebind all workers of a pool to the associated CPU
 * @pool: pool of interest
 *
 * @pool->cpu is coming online.  Rebind all workers to the CPU.
 */
static void rebind_workers(struct worker_pool *pool)
{
        struct worker *worker;

        lockdep_assert_held(&wq_pool_attach_mutex);

        /*
         * Restore CPU affinity of all workers.  As all idle workers should
         * be on the run-queue of the associated CPU before any local
         * wake-ups for concurrency management happen, restore CPU affinity
         * of all workers first and then clear UNBOUND.  As we're called
         * from CPU_ONLINE, the following shouldn't fail.
         */
        for_each_pool_worker(worker, pool) {
                kthread_set_per_cpu(worker->task, pool->cpu);
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
                                                  pool_allowed_cpus(pool)) < 0);
        }

        raw_spin_lock_irq(&pool->lock);

        pool->flags &= ~POOL_DISASSOCIATED;

        for_each_pool_worker(worker, pool) {
                unsigned int worker_flags = worker->flags;

                /*
                 * We want to clear UNBOUND but can't directly call
                 * worker_clr_flags() or adjust nr_running.  Atomically
                 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
                 * @worker will clear REBOUND using worker_clr_flags() when
                 * it initiates the next execution cycle thus restoring
                 * concurrency management.  Note that when or whether
                 * @worker clears REBOUND doesn't affect correctness.
                 *
                 * WRITE_ONCE() is necessary because @worker->flags may be
                 * tested without holding any lock in
                 * wq_worker_running().  Without it, NOT_RUNNING test may
                 * fail incorrectly leading to premature concurrency
                 * management operations.
                 */
                WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
                worker_flags |= WORKER_REBOUND;
                worker_flags &= ~WORKER_UNBOUND;
                WRITE_ONCE(worker->flags, worker_flags);
        }

        raw_spin_unlock_irq(&pool->lock);
}

/**
 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
 * @pool: unbound pool of interest
 * @cpu: the CPU which is coming up
 *
 * An unbound pool may end up with a cpumask which doesn't have any online
 * CPUs.  When a worker of such pool get scheduled, the scheduler resets
 * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
 * online CPU before, cpus_allowed of all its workers should be restored.
 */
static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
{
        static cpumask_t cpumask;
        struct worker *worker;

        lockdep_assert_held(&wq_pool_attach_mutex);

        /* is @cpu allowed for @pool? */
        if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
                return;

        cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);

        /* as we're called from CPU_ONLINE, the following shouldn't fail */
        for_each_pool_worker(worker, pool)
                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
}

int workqueue_prepare_cpu(unsigned int cpu)
{
        struct worker_pool *pool;

        for_each_cpu_worker_pool(pool, cpu) {
                if (pool->nr_workers)
                        continue;
                if (!create_worker(pool))
                        return -ENOMEM;
        }
        return 0;
}

int workqueue_online_cpu(unsigned int cpu)
{
        struct worker_pool *pool;
        struct workqueue_struct *wq;
        int pi;

        mutex_lock(&wq_pool_mutex);

        cpumask_set_cpu(cpu, wq_online_cpumask);

        for_each_pool(pool, pi) {
                /* BH pools aren't affected by hotplug */
                if (pool->flags & POOL_BH)
                        continue;

                mutex_lock(&wq_pool_attach_mutex);
                if (pool->cpu == cpu)
                        rebind_workers(pool);
                else if (pool->cpu < 0)
                        restore_unbound_workers_cpumask(pool, cpu);
                mutex_unlock(&wq_pool_attach_mutex);
        }

        /* update pod affinity of unbound workqueues */
        list_for_each_entry(wq, &workqueues, list) {
                struct workqueue_attrs *attrs = wq->unbound_attrs;

                if (attrs) {
                        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
                        int tcpu;

                        for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
                                unbound_wq_update_pwq(wq, tcpu);

                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, -1);
                        mutex_unlock(&wq->mutex);
                }
        }

        mutex_unlock(&wq_pool_mutex);
        return 0;
}

int workqueue_offline_cpu(unsigned int cpu)
{
        struct workqueue_struct *wq;

        /* unbinding per-cpu workers should happen on the local CPU */
        if (WARN_ON(cpu != smp_processor_id()))
                return -1;

        unbind_workers(cpu);

        /* update pod affinity of unbound workqueues */
        mutex_lock(&wq_pool_mutex);

        cpumask_clear_cpu(cpu, wq_online_cpumask);

        list_for_each_entry(wq, &workqueues, list) {
                struct workqueue_attrs *attrs = wq->unbound_attrs;

                if (attrs) {
                        const struct wq_pod_type *pt = wqattrs_pod_type(attrs);
                        int tcpu;

                        for_each_cpu(tcpu, pt->pod_cpus[pt->cpu_pod[cpu]])
                                unbound_wq_update_pwq(wq, tcpu);

                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, cpu);
                        mutex_unlock(&wq->mutex);
                }
        }
        mutex_unlock(&wq_pool_mutex);

        return 0;
}

struct work_for_cpu {
        struct work_struct work;
        long (*fn)(void *);
        void *arg;
        long ret;
};

static void work_for_cpu_fn(struct work_struct *work)
{
        struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);

        wfc->ret = wfc->fn(wfc->arg);
}

/**
 * work_on_cpu_key - run a function in thread context on a particular cpu
 * @cpu: the cpu to run on
 * @fn: the function to run
 * @arg: the function arg
 * @key: The lock class key for lock debugging purposes
 *
 * It is up to the caller to ensure that the cpu doesn't go offline.
 * The caller must not hold any locks which would prevent @fn from completing.
 *
 * Return: The value @fn returns.
 */
long work_on_cpu_key(int cpu, long (*fn)(void *),
                     void *arg, struct lock_class_key *key)
{
        struct work_for_cpu wfc = { .fn = fn, .arg = arg };

        INIT_WORK_ONSTACK_KEY(&wfc.work, work_for_cpu_fn, key);
        schedule_work_on(cpu, &wfc.work);
        flush_work(&wfc.work);
        destroy_work_on_stack(&wfc.work);
        return wfc.ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_key);

/**
 * work_on_cpu_safe_key - run a function in thread context on a particular cpu
 * @cpu: the cpu to run on
 * @fn:  the function to run
 * @arg: the function argument
 * @key: The lock class key for lock debugging purposes
 *
 * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
 * any locks which would prevent @fn from completing.
 *
 * Return: The value @fn returns.
 */
long work_on_cpu_safe_key(int cpu, long (*fn)(void *),
                          void *arg, struct lock_class_key *key)
{
        long ret = -ENODEV;

        cpus_read_lock();
        if (cpu_online(cpu))
                ret = work_on_cpu_key(cpu, fn, arg, key);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(work_on_cpu_safe_key);
#endif /* CONFIG_SMP */

#ifdef CONFIG_FREEZER

/**
 * freeze_workqueues_begin - begin freezing workqueues
 *
 * Start freezing workqueues.  After this function returns, all freezable
 * workqueues will queue new works to their inactive_works list instead of
 * pool->worklist.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void freeze_workqueues_begin(void)
{
        struct workqueue_struct *wq;

        mutex_lock(&wq_pool_mutex);

        WARN_ON_ONCE(workqueue_freezing);
        workqueue_freezing = true;

        list_for_each_entry(wq, &workqueues, list) {
                mutex_lock(&wq->mutex);
                wq_adjust_max_active(wq);
                mutex_unlock(&wq->mutex);
        }

        mutex_unlock(&wq_pool_mutex);
}

/**
 * freeze_workqueues_busy - are freezable workqueues still busy?
 *
 * Check whether freezing is complete.  This function must be called
 * between freeze_workqueues_begin() and thaw_workqueues().
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex.
 *
 * Return:
 * %true if some freezable workqueues are still busy.  %false if freezing
 * is complete.
 */
bool freeze_workqueues_busy(void)
{
        bool busy = false;
        struct workqueue_struct *wq;
        struct pool_workqueue *pwq;

        mutex_lock(&wq_pool_mutex);

        WARN_ON_ONCE(!workqueue_freezing);

        list_for_each_entry(wq, &workqueues, list) {
                if (!(wq->flags & WQ_FREEZABLE))
                        continue;
                /*
                 * nr_active is monotonically decreasing.  It's safe
                 * to peek without lock.
                 */
                rcu_read_lock();
                for_each_pwq(pwq, wq) {
                        WARN_ON_ONCE(pwq->nr_active < 0);
                        if (pwq->nr_active) {
                                busy = true;
                                rcu_read_unlock();
                                goto out_unlock;
                        }
                }
                rcu_read_unlock();
        }
out_unlock:
        mutex_unlock(&wq_pool_mutex);
        return busy;
}

/**
 * thaw_workqueues - thaw workqueues
 *
 * Thaw workqueues.  Normal queueing is restored and all collected
 * frozen works are transferred to their respective pool worklists.
 *
 * CONTEXT:
 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
 */
void thaw_workqueues(void)
{
        struct workqueue_struct *wq;

        mutex_lock(&wq_pool_mutex);

        if (!workqueue_freezing)
                goto out_unlock;

        workqueue_freezing = false;

        /* restore max_active and repopulate worklist */
        list_for_each_entry(wq, &workqueues, list) {
                mutex_lock(&wq->mutex);
                wq_adjust_max_active(wq);
                mutex_unlock(&wq->mutex);
        }

out_unlock:
        mutex_unlock(&wq_pool_mutex);
}
#endif /* CONFIG_FREEZER */

static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
{
        LIST_HEAD(ctxs);
        int ret = 0;
        struct workqueue_struct *wq;
        struct apply_wqattrs_ctx *ctx, *n;

        lockdep_assert_held(&wq_pool_mutex);

        list_for_each_entry(wq, &workqueues, list) {
                if (!(wq->flags & WQ_UNBOUND) || (wq->flags & __WQ_DESTROYING))
                        continue;

                ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
                if (IS_ERR(ctx)) {
                        ret = PTR_ERR(ctx);
                        break;
                }

                list_add_tail(&ctx->list, &ctxs);
        }

        list_for_each_entry_safe(ctx, n, &ctxs, list) {
                if (!ret)
                        apply_wqattrs_commit(ctx);
                apply_wqattrs_cleanup(ctx);
        }

        if (!ret) {
                mutex_lock(&wq_pool_attach_mutex);
                cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
                mutex_unlock(&wq_pool_attach_mutex);
        }
        return ret;
}

/**
 * workqueue_unbound_exclude_cpumask - Exclude given CPUs from unbound cpumask
 * @exclude_cpumask: the cpumask to be excluded from wq_unbound_cpumask
 *
 * This function can be called from cpuset code to provide a set of isolated
 * CPUs that should be excluded from wq_unbound_cpumask.
 */
int workqueue_unbound_exclude_cpumask(cpumask_var_t exclude_cpumask)
{
        cpumask_var_t cpumask;
        int ret = 0;

        if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
                return -ENOMEM;

        mutex_lock(&wq_pool_mutex);

        /*
         * If the operation fails, it will fall back to
         * wq_requested_unbound_cpumask which is initially set to
         * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten
         * by any subsequent write to workqueue/cpumask sysfs file.
         */
        if (!cpumask_andnot(cpumask, wq_requested_unbound_cpumask, exclude_cpumask))
                cpumask_copy(cpumask, wq_requested_unbound_cpumask);
        if (!cpumask_equal(cpumask, wq_unbound_cpumask))
                ret = workqueue_apply_unbound_cpumask(cpumask);

        /* Save the current isolated cpumask & export it via sysfs */
        if (!ret)
                cpumask_copy(wq_isolated_cpumask, exclude_cpumask);

        mutex_unlock(&wq_pool_mutex);
        free_cpumask_var(cpumask);
        return ret;
}

static int parse_affn_scope(const char *val)
{
        int i;

        for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) {
                if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i])))
                        return i;
        }
        return -EINVAL;
}

static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp)
{
        struct workqueue_struct *wq;
        int affn, cpu;

        affn = parse_affn_scope(val);
        if (affn < 0)
                return affn;
        if (affn == WQ_AFFN_DFL)
                return -EINVAL;

        cpus_read_lock();
        mutex_lock(&wq_pool_mutex);

        wq_affn_dfl = affn;

        list_for_each_entry(wq, &workqueues, list) {
                for_each_online_cpu(cpu)
                        unbound_wq_update_pwq(wq, cpu);
        }

        mutex_unlock(&wq_pool_mutex);
        cpus_read_unlock();

        return 0;
}

static int wq_affn_dfl_get(char *buffer, const struct kernel_param *kp)
{
        return scnprintf(buffer, PAGE_SIZE, "%s\n", wq_affn_names[wq_affn_dfl]);
}

static const struct kernel_param_ops wq_affn_dfl_ops = {
        .set        = wq_affn_dfl_set,
        .get        = wq_affn_dfl_get,
};

module_param_cb(default_affinity_scope, &wq_affn_dfl_ops, NULL, 0644);

#ifdef CONFIG_SYSFS
/*
 * Workqueues with WQ_SYSFS flag set is visible to userland via
 * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
 * following attributes.
 *
 *  per_cpu                RO bool        : whether the workqueue is per-cpu or unbound
 *  max_active                RW int        : maximum number of in-flight work items
 *
 * Unbound workqueues have the following extra attributes.
 *
 *  nice                RW int        : nice value of the workers
 *  cpumask                RW mask        : bitmask of allowed CPUs for the workers
 *  affinity_scope        RW str  : worker CPU affinity scope (cache, numa, none)
 *  affinity_strict        RW bool : worker CPU affinity is strict
 */
struct wq_device {
        struct workqueue_struct                *wq;
        struct device                        dev;
};

static struct workqueue_struct *dev_to_wq(struct device *dev)
{
        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);

        return wq_dev->wq;
}

static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
}
static DEVICE_ATTR_RO(per_cpu);

static ssize_t max_active_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
}

static ssize_t max_active_store(struct device *dev,
                                struct device_attribute *attr, const char *buf,
                                size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int val;

        if (sscanf(buf, "%d", &val) != 1 || val <= 0)
                return -EINVAL;

        workqueue_set_max_active(wq, val);
        return count;
}
static DEVICE_ATTR_RW(max_active);

static struct attribute *wq_sysfs_attrs[] = {
        &dev_attr_per_cpu.attr,
        &dev_attr_max_active.attr,
        NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs);

static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
                            char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
        mutex_unlock(&wq->mutex);

        return written;
}

/* prepare workqueue_attrs for sysfs store operations */
static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
{
        struct workqueue_attrs *attrs;

        lockdep_assert_held(&wq_pool_mutex);

        attrs = alloc_workqueue_attrs();
        if (!attrs)
                return NULL;

        copy_workqueue_attrs(attrs, wq->unbound_attrs);
        return attrs;
}

static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
                             const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int ret = -ENOMEM;

        apply_wqattrs_lock();

        attrs = wq_sysfs_prep_attrs(wq);
        if (!attrs)
                goto out_unlock;

        if (sscanf(buf, "%d", &attrs->nice) == 1 &&
            attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
                ret = apply_workqueue_attrs_locked(wq, attrs);
        else
                ret = -EINVAL;

out_unlock:
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_cpumask_show(struct device *dev,
                               struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
                            cpumask_pr_args(wq->unbound_attrs->cpumask));
        mutex_unlock(&wq->mutex);
        return written;
}

static ssize_t wq_cpumask_store(struct device *dev,
                                struct device_attribute *attr,
                                const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int ret = -ENOMEM;

        apply_wqattrs_lock();

        attrs = wq_sysfs_prep_attrs(wq);
        if (!attrs)
                goto out_unlock;

        ret = cpumask_parse(buf, attrs->cpumask);
        if (!ret)
                ret = apply_workqueue_attrs_locked(wq, attrs);

out_unlock:
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_affn_scope_show(struct device *dev,
                                  struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        int written;

        mutex_lock(&wq->mutex);
        if (wq->unbound_attrs->affn_scope == WQ_AFFN_DFL)
                written = scnprintf(buf, PAGE_SIZE, "%s (%s)\n",
                                    wq_affn_names[WQ_AFFN_DFL],
                                    wq_affn_names[wq_affn_dfl]);
        else
                written = scnprintf(buf, PAGE_SIZE, "%s\n",
                                    wq_affn_names[wq->unbound_attrs->affn_scope]);
        mutex_unlock(&wq->mutex);

        return written;
}

static ssize_t wq_affn_scope_store(struct device *dev,
                                   struct device_attribute *attr,
                                   const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int affn, ret = -ENOMEM;

        affn = parse_affn_scope(buf);
        if (affn < 0)
                return affn;

        apply_wqattrs_lock();
        attrs = wq_sysfs_prep_attrs(wq);
        if (attrs) {
                attrs->affn_scope = affn;
                ret = apply_workqueue_attrs_locked(wq, attrs);
        }
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static ssize_t wq_affinity_strict_show(struct device *dev,
                                       struct device_attribute *attr, char *buf)
{
        struct workqueue_struct *wq = dev_to_wq(dev);

        return scnprintf(buf, PAGE_SIZE, "%d\n",
                         wq->unbound_attrs->affn_strict);
}

static ssize_t wq_affinity_strict_store(struct device *dev,
                                        struct device_attribute *attr,
                                        const char *buf, size_t count)
{
        struct workqueue_struct *wq = dev_to_wq(dev);
        struct workqueue_attrs *attrs;
        int v, ret = -ENOMEM;

        if (sscanf(buf, "%d", &v) != 1)
                return -EINVAL;

        apply_wqattrs_lock();
        attrs = wq_sysfs_prep_attrs(wq);
        if (attrs) {
                attrs->affn_strict = (bool)v;
                ret = apply_workqueue_attrs_locked(wq, attrs);
        }
        apply_wqattrs_unlock();
        free_workqueue_attrs(attrs);
        return ret ?: count;
}

static struct device_attribute wq_sysfs_unbound_attrs[] = {
        __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
        __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
        __ATTR(affinity_scope, 0644, wq_affn_scope_show, wq_affn_scope_store),
        __ATTR(affinity_strict, 0644, wq_affinity_strict_show, wq_affinity_strict_store),
        __ATTR_NULL,
};

static const struct bus_type wq_subsys = {
        .name                                = "workqueue",
        .dev_groups                        = wq_sysfs_groups,
};

/**
 *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
 *  @cpumask: the cpumask to set
 *
 *  The low-level workqueues cpumask is a global cpumask that limits
 *  the affinity of all unbound workqueues.  This function check the @cpumask
 *  and apply it to all unbound workqueues and updates all pwqs of them.
 *
 *  Return:        0        - Success
 *                -EINVAL        - Invalid @cpumask
 *                -ENOMEM        - Failed to allocate memory for attrs or pwqs.
 */
static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
{
        int ret = -EINVAL;

        /*
         * Not excluding isolated cpus on purpose.
         * If the user wishes to include them, we allow that.
         */
        cpumask_and(cpumask, cpumask, cpu_possible_mask);
        if (!cpumask_empty(cpumask)) {
                ret = 0;
                apply_wqattrs_lock();
                if (!cpumask_equal(cpumask, wq_unbound_cpumask))
                        ret = workqueue_apply_unbound_cpumask(cpumask);
                if (!ret)
                        cpumask_copy(wq_requested_unbound_cpumask, cpumask);
                apply_wqattrs_unlock();
        }

        return ret;
}

static ssize_t __wq_cpumask_show(struct device *dev,
                struct device_attribute *attr, char *buf, cpumask_var_t mask)
{
        int written;

        mutex_lock(&wq_pool_mutex);
        written = scnprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
        mutex_unlock(&wq_pool_mutex);

        return written;
}

static ssize_t cpumask_requested_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_requested_unbound_cpumask);
}
static DEVICE_ATTR_RO(cpumask_requested);

static ssize_t cpumask_isolated_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_isolated_cpumask);
}
static DEVICE_ATTR_RO(cpumask_isolated);

static ssize_t cpumask_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        return __wq_cpumask_show(dev, attr, buf, wq_unbound_cpumask);
}

static ssize_t cpumask_store(struct device *dev,
                struct device_attribute *attr, const char *buf, size_t count)
{
        cpumask_var_t cpumask;
        int ret;

        if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
                return -ENOMEM;

        ret = cpumask_parse(buf, cpumask);
        if (!ret)
                ret = workqueue_set_unbound_cpumask(cpumask);

        free_cpumask_var(cpumask);
        return ret ? ret : count;
}
static DEVICE_ATTR_RW(cpumask);

static struct attribute *wq_sysfs_cpumask_attrs[] = {
        &dev_attr_cpumask.attr,
        &dev_attr_cpumask_requested.attr,
        &dev_attr_cpumask_isolated.attr,
        NULL,
};
ATTRIBUTE_GROUPS(wq_sysfs_cpumask);

static int __init wq_sysfs_init(void)
{
        return subsys_virtual_register(&wq_subsys, wq_sysfs_cpumask_groups);
}
core_initcall(wq_sysfs_init);

static void wq_device_release(struct device *dev)
{
        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);

        kfree(wq_dev);
}

/**
 * workqueue_sysfs_register - make a workqueue visible in sysfs
 * @wq: the workqueue to register
 *
 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
 * which is the preferred method.
 *
 * Workqueue user should use this function directly iff it wants to apply
 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
 * apply_workqueue_attrs() may race against userland updating the
 * attributes.
 *
 * Return: 0 on success, -errno on failure.
 */
int workqueue_sysfs_register(struct workqueue_struct *wq)
{
        struct wq_device *wq_dev;
        int ret;

        /*
         * Adjusting max_active breaks ordering guarantee.  Disallow exposing
         * ordered workqueues.
         */
        if (WARN_ON(wq->flags & __WQ_ORDERED))
                return -EINVAL;

        wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
        if (!wq_dev)
                return -ENOMEM;

        wq_dev->wq = wq;
        wq_dev->dev.bus = &wq_subsys;
        wq_dev->dev.release = wq_device_release;
        dev_set_name(&wq_dev->dev, "%s", wq->name);

        /*
         * unbound_attrs are created separately.  Suppress uevent until
         * everything is ready.
         */
        dev_set_uevent_suppress(&wq_dev->dev, true);

        ret = device_register(&wq_dev->dev);
        if (ret) {
                put_device(&wq_dev->dev);
                wq->wq_dev = NULL;
                return ret;
        }

        if (wq->flags & WQ_UNBOUND) {
                struct device_attribute *attr;

                for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
                        ret = device_create_file(&wq_dev->dev, attr);
                        if (ret) {
                                device_unregister(&wq_dev->dev);
                                wq->wq_dev = NULL;
                                return ret;
                        }
                }
        }

        dev_set_uevent_suppress(&wq_dev->dev, false);
        kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
        return 0;
}

/**
 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
 * @wq: the workqueue to unregister
 *
 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
 */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
{
        struct wq_device *wq_dev = wq->wq_dev;

        if (!wq->wq_dev)
                return;

        wq->wq_dev = NULL;
        device_unregister(&wq_dev->dev);
}
#else        /* CONFIG_SYSFS */
static void workqueue_sysfs_unregister(struct workqueue_struct *wq)        { }
#endif        /* CONFIG_SYSFS */

/*
 * Workqueue watchdog.
 *
 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
 * flush dependency, a concurrency managed work item which stays RUNNING
 * indefinitely.  Workqueue stalls can be very difficult to debug as the
 * usual warning mechanisms don't trigger and internal workqueue state is
 * largely opaque.
 *
 * Workqueue watchdog monitors all worker pools periodically and dumps
 * state if some pools failed to make forward progress for a while where
 * forward progress is defined as the first item on ->worklist changing.
 *
 * This mechanism is controlled through the kernel parameter
 * "workqueue.watchdog_thresh" which can be updated at runtime through the
 * corresponding sysfs parameter file.
 */
#ifdef CONFIG_WQ_WATCHDOG

static unsigned long wq_watchdog_thresh = 30;
static struct timer_list wq_watchdog_timer;

static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;

static unsigned int wq_panic_on_stall;
module_param_named(panic_on_stall, wq_panic_on_stall, uint, 0644);

/*
 * Show workers that might prevent the processing of pending work items.
 * The only candidates are CPU-bound workers in the running state.
 * Pending work items should be handled by another idle worker
 * in all other situations.
 */
static void show_cpu_pool_hog(struct worker_pool *pool)
{
        struct worker *worker;
        unsigned long irq_flags;
        int bkt;

        raw_spin_lock_irqsave(&pool->lock, irq_flags);

        hash_for_each(pool->busy_hash, bkt, worker, hentry) {
                if (task_is_running(worker->task)) {
                        /*
                         * Defer printing to avoid deadlocks in console
                         * drivers that queue work while holding locks
                         * also taken in their write paths.
                         */
                        printk_deferred_enter();

                        pr_info("pool %d:\n", pool->id);
                        sched_show_task(worker->task);

                        printk_deferred_exit();
                }
        }

        raw_spin_unlock_irqrestore(&pool->lock, irq_flags);
}

static void show_cpu_pools_hogs(void)
{
        struct worker_pool *pool;
        int pi;

        pr_info("Showing backtraces of running workers in stalled CPU-bound worker pools:\n");

        rcu_read_lock();

        for_each_pool(pool, pi) {
                if (pool->cpu_stall)
                        show_cpu_pool_hog(pool);

        }

        rcu_read_unlock();
}

static void panic_on_wq_watchdog(void)
{
        static unsigned int wq_stall;

        if (wq_panic_on_stall) {
                wq_stall++;
                BUG_ON(wq_stall >= wq_panic_on_stall);
        }
}

static void wq_watchdog_reset_touched(void)
{
        int cpu;

        wq_watchdog_touched = jiffies;
        for_each_possible_cpu(cpu)
                per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
}

static void wq_watchdog_timer_fn(struct timer_list *unused)
{
        unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
        bool lockup_detected = false;
        bool cpu_pool_stall = false;
        unsigned long now = jiffies;
        struct worker_pool *pool;
        int pi;

        if (!thresh)
                return;

        rcu_read_lock();

        for_each_pool(pool, pi) {
                unsigned long pool_ts, touched, ts;

                pool->cpu_stall = false;
                if (list_empty(&pool->worklist))
                        continue;

                /*
                 * If a virtual machine is stopped by the host it can look to
                 * the watchdog like a stall.
                 */
                kvm_check_and_clear_guest_paused();

                /* get the latest of pool and touched timestamps */
                if (pool->cpu >= 0)
                        touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
                else
                        touched = READ_ONCE(wq_watchdog_touched);
                pool_ts = READ_ONCE(pool->watchdog_ts);

                if (time_after(pool_ts, touched))
                        ts = pool_ts;
                else
                        ts = touched;

                /* did we stall? */
                if (time_after(now, ts + thresh)) {
                        lockup_detected = true;
                        if (pool->cpu >= 0 && !(pool->flags & POOL_BH)) {
                                pool->cpu_stall = true;
                                cpu_pool_stall = true;
                        }
                        pr_emerg("BUG: workqueue lockup - pool");
                        pr_cont_pool_info(pool);
                        pr_cont(" stuck for %us!\n",
                                jiffies_to_msecs(now - pool_ts) / 1000);
                }


        }

        rcu_read_unlock();

        if (lockup_detected)
                show_all_workqueues();

        if (cpu_pool_stall)
                show_cpu_pools_hogs();

        if (lockup_detected)
                panic_on_wq_watchdog();

        wq_watchdog_reset_touched();
        mod_timer(&wq_watchdog_timer, jiffies + thresh);
}

notrace void wq_watchdog_touch(int cpu)
{
        unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
        unsigned long touch_ts = READ_ONCE(wq_watchdog_touched);
        unsigned long now = jiffies;

        if (cpu >= 0)
                per_cpu(wq_watchdog_touched_cpu, cpu) = now;
        else
                WARN_ONCE(1, "%s should be called with valid CPU", __func__);

        /* Don't unnecessarily store to global cacheline */
        if (time_after(now, touch_ts + thresh / 4))
                WRITE_ONCE(wq_watchdog_touched, jiffies);
}

static void wq_watchdog_set_thresh(unsigned long thresh)
{
        wq_watchdog_thresh = 0;
        timer_delete_sync(&wq_watchdog_timer);

        if (thresh) {
                wq_watchdog_thresh = thresh;
                wq_watchdog_reset_touched();
                mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
        }
}

static int wq_watchdog_param_set_thresh(const char *val,
                                        const struct kernel_param *kp)
{
        unsigned long thresh;
        int ret;

        ret = kstrtoul(val, 0, &thresh);
        if (ret)
                return ret;

        if (system_wq)
                wq_watchdog_set_thresh(thresh);
        else
                wq_watchdog_thresh = thresh;

        return 0;
}

static const struct kernel_param_ops wq_watchdog_thresh_ops = {
        .set        = wq_watchdog_param_set_thresh,
        .get        = param_get_ulong,
};

module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
                0644);

static void wq_watchdog_init(void)
{
        timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
        wq_watchdog_set_thresh(wq_watchdog_thresh);
}

#else        /* CONFIG_WQ_WATCHDOG */

static inline void wq_watchdog_init(void) { }

#endif        /* CONFIG_WQ_WATCHDOG */

static void bh_pool_kick_normal(struct irq_work *irq_work)
{
        raise_softirq_irqoff(TASKLET_SOFTIRQ);
}

static void bh_pool_kick_highpri(struct irq_work *irq_work)
{
        raise_softirq_irqoff(HI_SOFTIRQ);
}

static void __init restrict_unbound_cpumask(const char *name, const struct cpumask *mask)
{
        if (!cpumask_intersects(wq_unbound_cpumask, mask)) {
                pr_warn("workqueue: Restricting unbound_cpumask (%*pb) with %s (%*pb) leaves no CPU, ignoring\n",
                        cpumask_pr_args(wq_unbound_cpumask), name, cpumask_pr_args(mask));
                return;
        }

        cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, mask);
}

static void __init init_cpu_worker_pool(struct worker_pool *pool, int cpu, int nice)
{
        BUG_ON(init_worker_pool(pool));
        pool->cpu = cpu;
        cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
        cpumask_copy(pool->attrs->__pod_cpumask, cpumask_of(cpu));
        pool->attrs->nice = nice;
        pool->attrs->affn_strict = true;
        pool->node = cpu_to_node(cpu);

        /* alloc pool ID */
        mutex_lock(&wq_pool_mutex);
        BUG_ON(worker_pool_assign_id(pool));
        mutex_unlock(&wq_pool_mutex);
}

/**
 * workqueue_init_early - early init for workqueue subsystem
 *
 * This is the first step of three-staged workqueue subsystem initialization and
 * invoked as soon as the bare basics - memory allocation, cpumasks and idr are
 * up. It sets up all the data structures and system workqueues and allows early
 * boot code to create workqueues and queue/cancel work items. Actual work item
 * execution starts only after kthreads can be created and scheduled right
 * before early initcalls.
 */
void __init workqueue_init_early(void)
{
        struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM];
        int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
        void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal,
                                                       bh_pool_kick_highpri };
        int i, cpu;

        BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));

        BUG_ON(!alloc_cpumask_var(&wq_online_cpumask, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
        BUG_ON(!alloc_cpumask_var(&wq_requested_unbound_cpumask, GFP_KERNEL));
        BUG_ON(!zalloc_cpumask_var(&wq_isolated_cpumask, GFP_KERNEL));

        cpumask_copy(wq_online_cpumask, cpu_online_mask);
        cpumask_copy(wq_unbound_cpumask, cpu_possible_mask);
        restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ));
        restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN));
        if (!cpumask_empty(&wq_cmdline_cpumask))
                restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask);

        cpumask_copy(wq_requested_unbound_cpumask, wq_unbound_cpumask);
        cpumask_andnot(wq_isolated_cpumask, cpu_possible_mask,
                                                housekeeping_cpumask(HK_TYPE_DOMAIN));
        pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);

        unbound_wq_update_pwq_attrs_buf = alloc_workqueue_attrs();
        BUG_ON(!unbound_wq_update_pwq_attrs_buf);

        /*
         * If nohz_full is enabled, set power efficient workqueue as unbound.
         * This allows workqueue items to be moved to HK CPUs.
         */
        if (housekeeping_enabled(HK_TYPE_TICK))
                wq_power_efficient = true;

        /* initialize WQ_AFFN_SYSTEM pods */
        pt->pod_cpus = kcalloc(1, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
        pt->pod_node = kcalloc(1, sizeof(pt->pod_node[0]), GFP_KERNEL);
        pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
        BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod);

        BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE));

        pt->nr_pods = 1;
        cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);
        pt->pod_node[0] = NUMA_NO_NODE;
        pt->cpu_pod[0] = 0;

        /* initialize BH and CPU pools */
        for_each_possible_cpu(cpu) {
                struct worker_pool *pool;

                i = 0;
                for_each_bh_worker_pool(pool, cpu) {
                        init_cpu_worker_pool(pool, cpu, std_nice[i]);
                        pool->flags |= POOL_BH;
                        init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);
                        i++;
                }

                i = 0;
                for_each_cpu_worker_pool(pool, cpu)
                        init_cpu_worker_pool(pool, cpu, std_nice[i++]);
        }

        /* create default unbound and ordered wq attrs */
        for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
                struct workqueue_attrs *attrs;

                BUG_ON(!(attrs = alloc_workqueue_attrs()));
                attrs->nice = std_nice[i];
                unbound_std_wq_attrs[i] = attrs;

                /*
                 * An ordered wq should have only one pwq as ordering is
                 * guaranteed by max_active which is enforced by pwqs.
                 */
                BUG_ON(!(attrs = alloc_workqueue_attrs()));
                attrs->nice = std_nice[i];
                attrs->ordered = true;
                ordered_wq_attrs[i] = attrs;
        }

        system_wq = alloc_workqueue("events", 0, 0);
        system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
        system_long_wq = alloc_workqueue("events_long", 0, 0);
        system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
                                            WQ_MAX_ACTIVE);
        system_freezable_wq = alloc_workqueue("events_freezable",
                                              WQ_FREEZABLE, 0);
        system_power_efficient_wq = alloc_workqueue("events_power_efficient",
                                              WQ_POWER_EFFICIENT, 0);
        system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",
                                              WQ_FREEZABLE | WQ_POWER_EFFICIENT,
                                              0);
        system_bh_wq = alloc_workqueue("events_bh", WQ_BH, 0);
        system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",
                                               WQ_BH | WQ_HIGHPRI, 0);
        BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
               !system_unbound_wq || !system_freezable_wq ||
               !system_power_efficient_wq ||
               !system_freezable_power_efficient_wq ||
               !system_bh_wq || !system_bh_highpri_wq);
}

static void __init wq_cpu_intensive_thresh_init(void)
{
        unsigned long thresh;
        unsigned long bogo;

        pwq_release_worker = kthread_run_worker(0, "pool_workqueue_release");
        BUG_ON(IS_ERR(pwq_release_worker));

        /* if the user set it to a specific value, keep it */
        if (wq_cpu_intensive_thresh_us != ULONG_MAX)
                return;

        /*
         * The default of 10ms is derived from the fact that most modern (as of
         * 2023) processors can do a lot in 10ms and that it's just below what
         * most consider human-perceivable. However, the kernel also runs on a
         * lot slower CPUs including microcontrollers where the threshold is way
         * too low.
         *
         * Let's scale up the threshold upto 1 second if BogoMips is below 4000.
         * This is by no means accurate but it doesn't have to be. The mechanism
         * is still useful even when the threshold is fully scaled up. Also, as
         * the reports would usually be applicable to everyone, some machines
         * operating on longer thresholds won't significantly diminish their
         * usefulness.
         */
        thresh = 10 * USEC_PER_MSEC;

        /* see init/calibrate.c for lpj -> BogoMIPS calculation */
        bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);
        if (bogo < 4000)
                thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC);

        pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",
                 loops_per_jiffy, bogo, thresh);

        wq_cpu_intensive_thresh_us = thresh;
}

/**
 * workqueue_init - bring workqueue subsystem fully online
 *
 * This is the second step of three-staged workqueue subsystem initialization
 * and invoked as soon as kthreads can be created and scheduled. Workqueues have
 * been created and work items queued on them, but there are no kworkers
 * executing the work items yet. Populate the worker pools with the initial
 * workers and enable future kworker creations.
 */
void __init workqueue_init(void)
{
        struct workqueue_struct *wq;
        struct worker_pool *pool;
        int cpu, bkt;

        wq_cpu_intensive_thresh_init();

        mutex_lock(&wq_pool_mutex);

        /*
         * Per-cpu pools created earlier could be missing node hint. Fix them
         * up. Also, create a rescuer for workqueues that requested it.
         */
        for_each_possible_cpu(cpu) {
                for_each_bh_worker_pool(pool, cpu)
                        pool->node = cpu_to_node(cpu);
                for_each_cpu_worker_pool(pool, cpu)
                        pool->node = cpu_to_node(cpu);
        }

        list_for_each_entry(wq, &workqueues, list) {
                WARN(init_rescuer(wq),
                     "workqueue: failed to create early rescuer for %s",
                     wq->name);
        }

        mutex_unlock(&wq_pool_mutex);

        /*
         * Create the initial workers. A BH pool has one pseudo worker that
         * represents the shared BH execution context and thus doesn't get
         * affected by hotplug events. Create the BH pseudo workers for all
         * possible CPUs here.
         */
        for_each_possible_cpu(cpu)
                for_each_bh_worker_pool(pool, cpu)
                        BUG_ON(!create_worker(pool));

        for_each_online_cpu(cpu) {
                for_each_cpu_worker_pool(pool, cpu) {
                        pool->flags &= ~POOL_DISASSOCIATED;
                        BUG_ON(!create_worker(pool));
                }
        }

        hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
                BUG_ON(!create_worker(pool));

        wq_online = true;
        wq_watchdog_init();
}

/*
 * Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to
 * @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique
 * and consecutive pod ID. The rest of @pt is initialized accordingly.
 */
static void __init init_pod_type(struct wq_pod_type *pt,
                                 bool (*cpus_share_pod)(int, int))
{
        int cur, pre, cpu, pod;

        pt->nr_pods = 0;

        /* init @pt->cpu_pod[] according to @cpus_share_pod() */
        pt->cpu_pod = kcalloc(nr_cpu_ids, sizeof(pt->cpu_pod[0]), GFP_KERNEL);
        BUG_ON(!pt->cpu_pod);

        for_each_possible_cpu(cur) {
                for_each_possible_cpu(pre) {
                        if (pre >= cur) {
                                pt->cpu_pod[cur] = pt->nr_pods++;
                                break;
                        }
                        if (cpus_share_pod(cur, pre)) {
                                pt->cpu_pod[cur] = pt->cpu_pod[pre];
                                break;
                        }
                }
        }

        /* init the rest to match @pt->cpu_pod[] */
        pt->pod_cpus = kcalloc(pt->nr_pods, sizeof(pt->pod_cpus[0]), GFP_KERNEL);
        pt->pod_node = kcalloc(pt->nr_pods, sizeof(pt->pod_node[0]), GFP_KERNEL);
        BUG_ON(!pt->pod_cpus || !pt->pod_node);

        for (pod = 0; pod < pt->nr_pods; pod++)
                BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL));

        for_each_possible_cpu(cpu) {
                cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);
                pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);
        }
}

static bool __init cpus_dont_share(int cpu0, int cpu1)
{
        return false;
}

static bool __init cpus_share_smt(int cpu0, int cpu1)
{
#ifdef CONFIG_SCHED_SMT
        return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
#else
        return false;
#endif
}

static bool __init cpus_share_numa(int cpu0, int cpu1)
{
        return cpu_to_node(cpu0) == cpu_to_node(cpu1);
}

/**
 * workqueue_init_topology - initialize CPU pods for unbound workqueues
 *
 * This is the third step of three-staged workqueue subsystem initialization and
 * invoked after SMP and topology information are fully initialized. It
 * initializes the unbound CPU pods accordingly.
 */
void __init workqueue_init_topology(void)
{
        struct workqueue_struct *wq;
        int cpu;

        init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);
        init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);
        init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);
        init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa);

        wq_topo_initialized = true;

        mutex_lock(&wq_pool_mutex);

        /*
         * Workqueues allocated earlier would have all CPUs sharing the default
         * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue
         * and CPU combinations to apply per-pod sharing.
         */
        list_for_each_entry(wq, &workqueues, list) {
                for_each_online_cpu(cpu)
                        unbound_wq_update_pwq(wq, cpu);
                if (wq->flags & WQ_UNBOUND) {
                        mutex_lock(&wq->mutex);
                        wq_update_node_max_active(wq, -1);
                        mutex_unlock(&wq->mutex);
                }
        }

        mutex_unlock(&wq_pool_mutex);
}

void __warn_flushing_systemwide_wq(void)
{
        pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");
        dump_stack();
}
EXPORT_SYMBOL(__warn_flushing_systemwide_wq);

static int __init workqueue_unbound_cpus_setup(char *str)
{
        if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {
                cpumask_clear(&wq_cmdline_cpumask);
                pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");
        }

        return 1;
}
__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);











































































































   25 






































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Statically sized hash table implementation
 * (C) 2012  Sasha Levin <levinsasha928@gmail.com>
 */

#ifndef _LINUX_HASHTABLE_H
#define _LINUX_HASHTABLE_H

#include <linux/list.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/hash.h>
#include <linux/rculist.h>

#define DEFINE_HASHTABLE(name, bits)                                                \
        struct hlist_head name[1 << (bits)] =                                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DEFINE_READ_MOSTLY_HASHTABLE(name, bits)                                \
        struct hlist_head name[1 << (bits)] __read_mostly =                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DECLARE_HASHTABLE(name, bits)                                           \
        struct hlist_head name[1 << (bits)]

#define HASH_SIZE(name) (ARRAY_SIZE(name))
#define HASH_BITS(name) ilog2(HASH_SIZE(name))

/* Use hash_32 when possible to allow for fast 32bit hashing in 64bit kernels. */
#define hash_min(val, bits)                                                        \
        (sizeof(val) <= 4 ? hash_32(val, bits) : hash_long(val, bits))

static inline void __hash_init(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                INIT_HLIST_HEAD(&ht[i]);
}

/**
 * hash_init - initialize a hash table
 * @hashtable: hashtable to be initialized
 *
 * Calculates the size of the hashtable from the given parameter, otherwise
 * same as hash_init_size.
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_init(hashtable) __hash_init(hashtable, HASH_SIZE(hashtable))

/**
 * hash_add - add an object to a hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add(hashtable, node, key)                                                \
        hlist_add_head(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_add_rcu - add an object to a rcu enabled hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add_rcu(hashtable, node, key)                                        \
        hlist_add_head_rcu(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_hashed - check whether an object is in any hashtable
 * @node: the &struct hlist_node of the object to be checked
 */
static inline bool hash_hashed(struct hlist_node *node)
{
        return !hlist_unhashed(node);
}

static inline bool __hash_empty(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                if (!hlist_empty(&ht[i]))
                        return false;

        return true;
}

/**
 * hash_empty - check whether a hashtable is empty
 * @hashtable: hashtable to check
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_empty(hashtable) __hash_empty(hashtable, HASH_SIZE(hashtable))

/**
 * hash_del - remove an object from a hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del(struct hlist_node *node)
{
        hlist_del_init(node);
}

/**
 * hash_del_rcu - remove an object from a rcu enabled hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del_rcu(struct hlist_node *node)
{
        hlist_del_init_rcu(node);
}

/**
 * hash_for_each - iterate over a hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each(name, bkt, obj, member)                                \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry(obj, &name[bkt], member)

/**
 * hash_for_each_rcu - iterate over a rcu enabled hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_rcu(name, bkt, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_rcu(obj, &name[bkt], member)

/**
 * hash_for_each_safe - iterate over a hashtable safe against removal of
 * hash entry
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @tmp: a &struct hlist_node used for temporary storage
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_safe(name, bkt, tmp, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_safe(obj, tmp, &name[bkt], member)

/**
 * hash_for_each_possible - iterate over all possible objects hashing to the
 * same bucket
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible(name, obj, member, key)                        \
        hlist_for_each_entry(obj, &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_rcu - iterate over all possible objects hashing to the
 * same bucket in an rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_rcu(name, obj, member, key, cond...)        \
        hlist_for_each_entry_rcu(obj, &name[hash_min(key, HASH_BITS(name))],\
                member, ## cond)

/**
 * hash_for_each_possible_rcu_notrace - iterate over all possible objects hashing
 * to the same bucket in an rcu enabled hashtable in a rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 *
 * This is the same as hash_for_each_possible_rcu() except that it does
 * not do any RCU debugging or tracing.
 */
#define hash_for_each_possible_rcu_notrace(name, obj, member, key) \
        hlist_for_each_entry_rcu_notrace(obj, \
                &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_safe - iterate over all possible objects hashing to the
 * same bucket safe against removals
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @tmp: a &struct hlist_node used for temporary storage
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_safe(name, obj, tmp, member, key)        \
        hlist_for_each_entry_safe(obj, tmp,\
                &name[hash_min(key, HASH_BITS(name))], member)


#endif





































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   29 







   29 
   29 








   29 














































































   29 








   29 















   29 





   29 



















































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2022 Linutronix GmbH, John Ogness
// Copyright (C) 2022 Intel, Thomas Gleixner

#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/console.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/irqflags.h>
#include <linux/kthread.h>
#include <linux/minmax.h>
#include <linux/percpu.h>
#include <linux/preempt.h>
#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/stddef.h>
#include <linux/string.h>
#include <linux/types.h>
#include "internal.h"
#include "printk_ringbuffer.h"
/*
 * Printk console printing implementation for consoles which does not depend
 * on the legacy style console_lock mechanism.
 *
 * The state of the console is maintained in the "nbcon_state" atomic
 * variable.
 *
 * The console is locked when:
 *
 *   - The 'prio' field contains the priority of the context that owns the
 *     console. Only higher priority contexts are allowed to take over the
 *     lock. A value of 0 (NBCON_PRIO_NONE) means the console is not locked.
 *
 *   - The 'cpu' field denotes on which CPU the console is locked. It is used
 *     to prevent busy waiting on the same CPU. Also it informs the lock owner
 *     that it has lost the lock in a more complex scenario when the lock was
 *     taken over by a higher priority context, released, and taken on another
 *     CPU with the same priority as the interrupted owner.
 *
 * The acquire mechanism uses a few more fields:
 *
 *   - The 'req_prio' field is used by the handover approach to make the
 *     current owner aware that there is a context with a higher priority
 *     waiting for the friendly handover.
 *
 *   - The 'unsafe' field allows to take over the console in a safe way in the
 *     middle of emitting a message. The field is set only when accessing some
 *     shared resources or when the console device is manipulated. It can be
 *     cleared, for example, after emitting one character when the console
 *     device is in a consistent state.
 *
 *   - The 'unsafe_takeover' field is set when a hostile takeover took the
 *     console in an unsafe state. The console will stay in the unsafe state
 *     until re-initialized.
 *
 * The acquire mechanism uses three approaches:
 *
 *   1) Direct acquire when the console is not owned or is owned by a lower
 *      priority context and is in a safe state.
 *
 *   2) Friendly handover mechanism uses a request/grant handshake. It is used
 *      when the current owner has lower priority and the console is in an
 *      unsafe state.
 *
 *      The requesting context:
 *
 *        a) Sets its priority into the 'req_prio' field.
 *
 *        b) Waits (with a timeout) for the owning context to unlock the
 *           console.
 *
 *        c) Takes the lock and clears the 'req_prio' field.
 *
 *      The owning context:
 *
 *        a) Observes the 'req_prio' field set on exit from the unsafe
 *           console state.
 *
 *        b) Gives up console ownership by clearing the 'prio' field.
 *
 *   3) Unsafe hostile takeover allows to take over the lock even when the
 *      console is an unsafe state. It is used only in panic() by the final
 *      attempt to flush consoles in a try and hope mode.
 *
 *      Note that separate record buffers are used in panic(). As a result,
 *      the messages can be read and formatted without any risk even after
 *      using the hostile takeover in unsafe state.
 *
 * The release function simply clears the 'prio' field.
 *
 * All operations on @console::nbcon_state are atomic cmpxchg based to
 * handle concurrency.
 *
 * The acquire/release functions implement only minimal policies:
 *
 *   - Preference for higher priority contexts.
 *   - Protection of the panic CPU.
 *
 * All other policy decisions must be made at the call sites:
 *
 *   - What is marked as an unsafe section.
 *   - Whether to spin-wait if there is already an owner and the console is
 *     in an unsafe state.
 *   - Whether to attempt an unsafe hostile takeover.
 *
 * The design allows to implement the well known:
 *
 *     acquire()
 *     output_one_printk_record()
 *     release()
 *
 * The output of one printk record might be interrupted with a higher priority
 * context. The new owner is supposed to reprint the entire interrupted record
 * from scratch.
 */

/**
 * nbcon_state_set - Helper function to set the console state
 * @con:        Console to update
 * @new:        The new state to write
 *
 * Only to be used when the console is not yet or no longer visible in the
 * system. Otherwise use nbcon_state_try_cmpxchg().
 */
static inline void nbcon_state_set(struct console *con, struct nbcon_state *new)
{
        atomic_set(&ACCESS_PRIVATE(con, nbcon_state), new->atom);
}

/**
 * nbcon_state_read - Helper function to read the console state
 * @con:        Console to read
 * @state:        The state to store the result
 */
static inline void nbcon_state_read(struct console *con, struct nbcon_state *state)
{
        state->atom = atomic_read(&ACCESS_PRIVATE(con, nbcon_state));
}

/**
 * nbcon_state_try_cmpxchg() - Helper function for atomic_try_cmpxchg() on console state
 * @con:        Console to update
 * @cur:        Old/expected state
 * @new:        New state
 *
 * Return: True on success. False on fail and @cur is updated.
 */
static inline bool nbcon_state_try_cmpxchg(struct console *con, struct nbcon_state *cur,
                                           struct nbcon_state *new)
{
        return atomic_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_state), &cur->atom, new->atom);
}

/**
 * nbcon_seq_read - Read the current console sequence
 * @con:        Console to read the sequence of
 *
 * Return:        Sequence number of the next record to print on @con.
 */
u64 nbcon_seq_read(struct console *con)
{
        unsigned long nbcon_seq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_seq));

        return __ulseq_to_u64seq(prb, nbcon_seq);
}

/**
 * nbcon_seq_force - Force console sequence to a specific value
 * @con:        Console to work on
 * @seq:        Sequence number value to set
 *
 * Only to be used during init (before registration) or in extreme situations
 * (such as panic with CONSOLE_REPLAY_ALL).
 */
void nbcon_seq_force(struct console *con, u64 seq)
{
        /*
         * If the specified record no longer exists, the oldest available record
         * is chosen. This is especially important on 32bit systems because only
         * the lower 32 bits of the sequence number are stored. The upper 32 bits
         * are derived from the sequence numbers available in the ringbuffer.
         */
        u64 valid_seq = max_t(u64, seq, prb_first_valid_seq(prb));

        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), __u64seq_to_ulseq(valid_seq));
}

/**
 * nbcon_seq_try_update - Try to update the console sequence number
 * @ctxt:        Pointer to an acquire context that contains
 *                all information about the acquire mode
 * @new_seq:        The new sequence number to set
 *
 * @ctxt->seq is updated to the new value of @con::nbcon_seq (expanded to
 * the 64bit value). This could be a different value than @new_seq if
 * nbcon_seq_force() was used or the current context no longer owns the
 * console. In the later case, it will stop printing anyway.
 */
static void nbcon_seq_try_update(struct nbcon_context *ctxt, u64 new_seq)
{
        unsigned long nbcon_seq = __u64seq_to_ulseq(ctxt->seq);
        struct console *con = ctxt->console;

        if (atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_seq), &nbcon_seq,
                                    __u64seq_to_ulseq(new_seq))) {
                ctxt->seq = new_seq;
        } else {
                ctxt->seq = nbcon_seq_read(con);
        }
}

/**
 * nbcon_context_try_acquire_direct - Try to acquire directly
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * Acquire the console when it is released. Also acquire the console when
 * the current owner has a lower priority and the console is in a safe state.
 *
 * Return:        0 on success. Otherwise, an error code on failure. Also @cur
 *                is updated to the latest state when failed to modify it.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is not the panic CPU.
 *                        Or the current owner or waiter has the same or higher
 *                        priority. No acquire method can be successful in
 *                        this case.
 *
 *        -EBUSY:                The current owner has a lower priority but the console
 *                        in an unsafe state. The caller should try using
 *                        the handover acquire method.
 */
static int nbcon_context_try_acquire_direct(struct nbcon_context *ctxt,
                                            struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        do {
                /*
                 * Panic does not imply that the console is owned. However, it
                 * is critical that non-panic CPUs during panic are unable to
                 * acquire ownership in order to satisfy the assumptions of
                 * nbcon_waiter_matches(). In particular, the assumption that
                 * lower priorities are ignored during panic.
                 */
                if (other_cpu_in_panic())
                        return -EPERM;

                if (ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio)
                        return -EPERM;

                if (cur->unsafe)
                        return -EBUSY;

                /*
                 * The console should never be safe for a direct acquire
                 * if an unsafe hostile takeover has ever happened.
                 */
                WARN_ON_ONCE(cur->unsafe_takeover);

                new.atom = cur->atom;
                new.prio        = ctxt->prio;
                new.req_prio        = NBCON_PRIO_NONE;
                new.unsafe        = cur->unsafe_takeover;
                new.cpu                = cpu;

        } while (!nbcon_state_try_cmpxchg(con, cur, &new));

        return 0;
}

static bool nbcon_waiter_matches(struct nbcon_state *cur, int expected_prio)
{
        /*
         * The request context is well defined by the @req_prio because:
         *
         * - Only a context with a priority higher than the owner can become
         *   a waiter.
         * - Only a context with a priority higher than the waiter can
         *   directly take over the request.
         * - There are only three priorities.
         * - Only one CPU is allowed to request PANIC priority.
         * - Lower priorities are ignored during panic() until reboot.
         *
         * As a result, the following scenario is *not* possible:
         *
         * 1. This context is currently a waiter.
         * 2. Another context with a higher priority than this context
         *    directly takes ownership.
         * 3. The higher priority context releases the ownership.
         * 4. Another lower priority context takes the ownership.
         * 5. Another context with the same priority as this context
         *    creates a request and starts waiting.
         *
         * Event #1 implies this context is EMERGENCY.
         * Event #2 implies the new context is PANIC.
         * Event #3 occurs when panic() has flushed the console.
         * Events #4 and #5 are not possible due to the other_cpu_in_panic()
         * check in nbcon_context_try_acquire_direct().
         */

        return (cur->req_prio == expected_prio);
}

/**
 * nbcon_context_try_acquire_requested - Try to acquire after having
 *                                         requested a handover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * This is a helper function for nbcon_context_try_acquire_handover().
 * It is called when the console is in an unsafe state. The current
 * owner will release the console on exit from the unsafe region.
 *
 * Return:        0 on success and @cur is updated to the new console state.
 *                Otherwise an error code on failure.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is not the panic CPU
 *                        or this context is no longer the waiter.
 *
 *        -EBUSY:                The console is still locked. The caller should
 *                        continue waiting.
 *
 * Note: The caller must still remove the request when an error has occurred
 *       except when this context is no longer the waiter.
 */
static int nbcon_context_try_acquire_requested(struct nbcon_context *ctxt,
                                               struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        /* Note that the caller must still remove the request! */
        if (other_cpu_in_panic())
                return -EPERM;

        /*
         * Note that the waiter will also change if there was an unsafe
         * hostile takeover.
         */
        if (!nbcon_waiter_matches(cur, ctxt->prio))
                return -EPERM;

        /* If still locked, caller should continue waiting. */
        if (cur->prio != NBCON_PRIO_NONE)
                return -EBUSY;

        /*
         * The previous owner should have never released ownership
         * in an unsafe region.
         */
        WARN_ON_ONCE(cur->unsafe);

        new.atom = cur->atom;
        new.prio        = ctxt->prio;
        new.req_prio        = NBCON_PRIO_NONE;
        new.unsafe        = cur->unsafe_takeover;
        new.cpu                = cpu;

        if (!nbcon_state_try_cmpxchg(con, cur, &new)) {
                /*
                 * The acquire could fail only when it has been taken
                 * over by a higher priority context.
                 */
                WARN_ON_ONCE(nbcon_waiter_matches(cur, ctxt->prio));
                return -EPERM;
        }

        /* Handover success. This context now owns the console. */
        return 0;
}

/**
 * nbcon_context_try_acquire_handover - Try to acquire via handover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * The function must be called only when the context has higher priority
 * than the current owner and the console is in an unsafe state.
 * It is the case when nbcon_context_try_acquire_direct() returns -EBUSY.
 *
 * The function sets "req_prio" field to make the current owner aware of
 * the request. Then it waits until the current owner releases the console,
 * or an even higher context takes over the request, or timeout expires.
 *
 * The current owner checks the "req_prio" field on exit from the unsafe
 * region and releases the console. It does not touch the "req_prio" field
 * so that the console stays reserved for the waiter.
 *
 * Return:        0 on success. Otherwise, an error code on failure. Also @cur
 *                is updated to the latest state when failed to modify it.
 *
 * Errors:
 *
 *        -EPERM:                A panic is in progress and this is not the panic CPU.
 *                        Or a higher priority context has taken over the
 *                        console or the handover request.
 *
 *        -EBUSY:                The current owner is on the same CPU so that the hand
 *                        shake could not work. Or the current owner is not
 *                        willing to wait (zero timeout). Or the console does
 *                        not enter the safe state before timeout passed. The
 *                        caller might still use the unsafe hostile takeover
 *                        when allowed.
 *
 *        -EAGAIN:        @cur has changed when creating the handover request.
 *                        The caller should retry with direct acquire.
 */
static int nbcon_context_try_acquire_handover(struct nbcon_context *ctxt,
                                              struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;
        int timeout;
        int request_err = -EBUSY;

        /*
         * Check that the handover is called when the direct acquire failed
         * with -EBUSY.
         */
        WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
        WARN_ON_ONCE(!cur->unsafe);

        /* Handover is not possible on the same CPU. */
        if (cur->cpu == cpu)
                return -EBUSY;

        /*
         * Console stays unsafe after an unsafe takeover until re-initialized.
         * Waiting is not going to help in this case.
         */
        if (cur->unsafe_takeover)
                return -EBUSY;

        /* Is the caller willing to wait? */
        if (ctxt->spinwait_max_us == 0)
                return -EBUSY;

        /*
         * Setup a request for the handover. The caller should try to acquire
         * the console directly when the current state has been modified.
         */
        new.atom = cur->atom;
        new.req_prio = ctxt->prio;
        if (!nbcon_state_try_cmpxchg(con, cur, &new))
                return -EAGAIN;

        cur->atom = new.atom;

        /* Wait until there is no owner and then acquire the console. */
        for (timeout = ctxt->spinwait_max_us; timeout >= 0; timeout--) {
                /* On successful acquire, this request is cleared. */
                request_err = nbcon_context_try_acquire_requested(ctxt, cur);
                if (!request_err)
                        return 0;

                /*
                 * If the acquire should be aborted, it must be ensured
                 * that the request is removed before returning to caller.
                 */
                if (request_err == -EPERM)
                        break;

                udelay(1);

                /* Re-read the state because some time has passed. */
                nbcon_state_read(con, cur);
        }

        /* Timed out or aborted. Carefully remove handover request. */
        do {
                /*
                 * No need to remove request if there is a new waiter. This
                 * can only happen if a higher priority context has taken over
                 * the console or the handover request.
                 */
                if (!nbcon_waiter_matches(cur, ctxt->prio))
                        return -EPERM;

                /* Unset request for handover. */
                new.atom = cur->atom;
                new.req_prio = NBCON_PRIO_NONE;
                if (nbcon_state_try_cmpxchg(con, cur, &new)) {
                        /*
                         * Request successfully unset. Report failure of
                         * acquiring via handover.
                         */
                        cur->atom = new.atom;
                        return request_err;
                }

                /*
                 * Unable to remove request. Try to acquire in case
                 * the owner has released the lock.
                 */
        } while (nbcon_context_try_acquire_requested(ctxt, cur));

        /* Lucky timing. The acquire succeeded while removing the request. */
        return 0;
}

/**
 * nbcon_context_try_acquire_hostile - Acquire via unsafe hostile takeover
 * @ctxt:        The context of the caller
 * @cur:        The current console state
 *
 * Acquire the console even in the unsafe state.
 *
 * It can be permitted by setting the 'allow_unsafe_takeover' field only
 * by the final attempt to flush messages in panic().
 *
 * Return:        0 on success. -EPERM when not allowed by the context.
 */
static int nbcon_context_try_acquire_hostile(struct nbcon_context *ctxt,
                                             struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state new;

        if (!ctxt->allow_unsafe_takeover)
                return -EPERM;

        /* Ensure caller is allowed to perform unsafe hostile takeovers. */
        if (WARN_ON_ONCE(ctxt->prio != NBCON_PRIO_PANIC))
                return -EPERM;

        /*
         * Check that try_acquire_direct() and try_acquire_handover() returned
         * -EBUSY in the right situation.
         */
        WARN_ON_ONCE(ctxt->prio <= cur->prio || ctxt->prio <= cur->req_prio);
        WARN_ON_ONCE(cur->unsafe != true);

        do {
                new.atom = cur->atom;
                new.cpu                        = cpu;
                new.prio                = ctxt->prio;
                new.unsafe                |= cur->unsafe_takeover;
                new.unsafe_takeover        |= cur->unsafe;

        } while (!nbcon_state_try_cmpxchg(con, cur, &new));

        return 0;
}

static struct printk_buffers panic_nbcon_pbufs;

/**
 * nbcon_context_try_acquire - Try to acquire nbcon console
 * @ctxt:        The context of the caller
 *
 * Context:        Under @ctxt->con->device_lock() or local_irq_save().
 * Return:        True if the console was acquired. False otherwise.
 *
 * If the caller allowed an unsafe hostile takeover, on success the
 * caller should check the current console state to see if it is
 * in an unsafe state. Otherwise, on success the caller may assume
 * the console is not in an unsafe state.
 */
static bool nbcon_context_try_acquire(struct nbcon_context *ctxt)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        int err;

        nbcon_state_read(con, &cur);
try_again:
        err = nbcon_context_try_acquire_direct(ctxt, &cur);
        if (err != -EBUSY)
                goto out;

        err = nbcon_context_try_acquire_handover(ctxt, &cur);
        if (err == -EAGAIN)
                goto try_again;
        if (err != -EBUSY)
                goto out;

        err = nbcon_context_try_acquire_hostile(ctxt, &cur);
out:
        if (err)
                return false;

        /* Acquire succeeded. */

        /* Assign the appropriate buffer for this context. */
        if (atomic_read(&panic_cpu) == cpu)
                ctxt->pbufs = &panic_nbcon_pbufs;
        else
                ctxt->pbufs = con->pbufs;

        /* Set the record sequence for this context to print. */
        ctxt->seq = nbcon_seq_read(ctxt->console);

        return true;
}

static bool nbcon_owner_matches(struct nbcon_state *cur, int expected_cpu,
                                int expected_prio)
{
        /*
         * A similar function, nbcon_waiter_matches(), only deals with
         * EMERGENCY and PANIC priorities. However, this function must also
         * deal with the NORMAL priority, which requires additional checks
         * and constraints.
         *
         * For the case where preemption and interrupts are disabled, it is
         * enough to also verify that the owning CPU has not changed.
         *
         * For the case where preemption or interrupts are enabled, an
         * external synchronization method *must* be used. In particular,
         * the driver-specific locking mechanism used in device_lock()
         * (including disabling migration) should be used. It prevents
         * scenarios such as:
         *
         * 1. [Task A] owns a context with NBCON_PRIO_NORMAL on [CPU X] and
         *    is scheduled out.
         * 2. Another context takes over the lock with NBCON_PRIO_EMERGENCY
         *    and releases it.
         * 3. [Task B] acquires a context with NBCON_PRIO_NORMAL on [CPU X]
         *    and is scheduled out.
         * 4. [Task A] gets running on [CPU X] and sees that the console is
         *    still owned by a task on [CPU X] with NBON_PRIO_NORMAL. Thus
         *    [Task A] thinks it is the owner when it is not.
         */

        if (cur->prio != expected_prio)
                return false;

        if (cur->cpu != expected_cpu)
                return false;

        return true;
}

/**
 * nbcon_context_release - Release the console
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 */
static void nbcon_context_release(struct nbcon_context *ctxt)
{
        unsigned int cpu = smp_processor_id();
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        struct nbcon_state new;

        nbcon_state_read(con, &cur);

        do {
                if (!nbcon_owner_matches(&cur, cpu, ctxt->prio))
                        break;

                new.atom = cur.atom;
                new.prio = NBCON_PRIO_NONE;

                /*
                 * If @unsafe_takeover is set, it is kept set so that
                 * the state remains permanently unsafe.
                 */
                new.unsafe |= cur.unsafe_takeover;

        } while (!nbcon_state_try_cmpxchg(con, &cur, &new));

        ctxt->pbufs = NULL;
}

/**
 * nbcon_context_can_proceed - Check whether ownership can proceed
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 * @cur:        The current console state
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * Must be invoked when entering the unsafe state to make sure that it still
 * owns the lock. Also must be invoked when exiting the unsafe context
 * to eventually free the lock for a higher priority context which asked
 * for the friendly handover.
 *
 * It can be called inside an unsafe section when the console is just
 * temporary in safe state instead of exiting and entering the unsafe
 * state.
 *
 * Also it can be called in the safe context before doing an expensive
 * safe operation. It does not make sense to do the operation when
 * a higher priority context took the lock.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
static bool nbcon_context_can_proceed(struct nbcon_context *ctxt, struct nbcon_state *cur)
{
        unsigned int cpu = smp_processor_id();

        /* Make sure this context still owns the console. */
        if (!nbcon_owner_matches(cur, cpu, ctxt->prio))
                return false;

        /* The console owner can proceed if there is no waiter. */
        if (cur->req_prio == NBCON_PRIO_NONE)
                return true;

        /*
         * A console owner within an unsafe region is always allowed to
         * proceed, even if there are waiters. It can perform a handover
         * when exiting the unsafe region. Otherwise the waiter will
         * need to perform an unsafe hostile takeover.
         */
        if (cur->unsafe)
                return true;

        /* Waiters always have higher priorities than owners. */
        WARN_ON_ONCE(cur->req_prio <= cur->prio);

        /*
         * Having a safe point for take over and eventually a few
         * duplicated characters or a full line is way better than a
         * hostile takeover. Post processing can take care of the garbage.
         * Release and hand over.
         */
        nbcon_context_release(ctxt);

        /*
         * It is not clear whether the waiter really took over ownership. The
         * outermost callsite must make the final decision whether console
         * ownership is needed for it to proceed. If yes, it must reacquire
         * ownership (possibly hostile) before carefully proceeding.
         *
         * The calling context no longer owns the console so go back all the
         * way instead of trying to implement reacquire heuristics in tons of
         * places.
         */
        return false;
}

/**
 * nbcon_can_proceed - Check whether ownership can proceed
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * It is used in nbcon_enter_unsafe() to make sure that it still owns the
 * lock. Also it is used in nbcon_exit_unsafe() to eventually free the lock
 * for a higher priority context which asked for the friendly handover.
 *
 * It can be called inside an unsafe section when the console is just
 * temporary in safe state instead of exiting and entering the unsafe state.
 *
 * Also it can be called in the safe context before doing an expensive safe
 * operation. It does not make sense to do the operation when a higher
 * priority context took the lock.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_can_proceed(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        struct nbcon_state cur;

        nbcon_state_read(con, &cur);

        return nbcon_context_can_proceed(ctxt, &cur);
}
EXPORT_SYMBOL_GPL(nbcon_can_proceed);

#define nbcon_context_enter_unsafe(c)        __nbcon_context_update_unsafe(c, true)
#define nbcon_context_exit_unsafe(c)        __nbcon_context_update_unsafe(c, false)

/**
 * __nbcon_context_update_unsafe - Update the unsafe bit in @con->nbcon_state
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 * @unsafe:        The new value for the unsafe bit
 *
 * Return:        True if the unsafe state was updated and this context still
 *                owns the console. Otherwise false if ownership was handed
 *                over or taken.
 *
 * This function allows console owners to modify the unsafe status of the
 * console.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 *
 * Internal helper to avoid duplicated code.
 */
static bool __nbcon_context_update_unsafe(struct nbcon_context *ctxt, bool unsafe)
{
        struct console *con = ctxt->console;
        struct nbcon_state cur;
        struct nbcon_state new;

        nbcon_state_read(con, &cur);

        do {
                /*
                 * The unsafe bit must not be cleared if an
                 * unsafe hostile takeover has occurred.
                 */
                if (!unsafe && cur.unsafe_takeover)
                        goto out;

                if (!nbcon_context_can_proceed(ctxt, &cur))
                        return false;

                new.atom = cur.atom;
                new.unsafe = unsafe;
        } while (!nbcon_state_try_cmpxchg(con, &cur, &new));

        cur.atom = new.atom;
out:
        return nbcon_context_can_proceed(ctxt, &cur);
}

static void nbcon_write_context_set_buf(struct nbcon_write_context *wctxt,
                                        char *buf, unsigned int len)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        struct nbcon_state cur;

        wctxt->outbuf = buf;
        wctxt->len = len;
        nbcon_state_read(con, &cur);
        wctxt->unsafe_takeover = cur.unsafe_takeover;
}

/**
 * nbcon_enter_unsafe - Enter an unsafe region in the driver
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_enter_unsafe(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        bool is_owner;

        is_owner = nbcon_context_enter_unsafe(ctxt);
        if (!is_owner)
                nbcon_write_context_set_buf(wctxt, NULL, 0);
        return is_owner;
}
EXPORT_SYMBOL_GPL(nbcon_enter_unsafe);

/**
 * nbcon_exit_unsafe - Exit an unsafe region in the driver
 * @wctxt:        The write context that was handed to the write function
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context.
 */
bool nbcon_exit_unsafe(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        bool ret;

        ret = nbcon_context_exit_unsafe(ctxt);
        if (!ret)
                nbcon_write_context_set_buf(wctxt, NULL, 0);
        return ret;
}
EXPORT_SYMBOL_GPL(nbcon_exit_unsafe);

/**
 * nbcon_reacquire_nobuf - Reacquire a console after losing ownership
 *                                while printing
 * @wctxt:        The write context that was handed to the write callback
 *
 * Since ownership can be lost at any time due to handover or takeover, a
 * printing context _must_ be prepared to back out immediately and
 * carefully. However, there are scenarios where the printing context must
 * reacquire ownership in order to finalize or revert hardware changes.
 *
 * This function allows a printing context to reacquire ownership using the
 * same priority as its previous ownership.
 *
 * Note that after a successful reacquire the printing context will have no
 * output buffer because that has been lost. This function cannot be used to
 * resume printing.
 */
void nbcon_reacquire_nobuf(struct nbcon_write_context *wctxt)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);

        while (!nbcon_context_try_acquire(ctxt))
                cpu_relax();

        nbcon_write_context_set_buf(wctxt, NULL, 0);
}
EXPORT_SYMBOL_GPL(nbcon_reacquire_nobuf);

/**
 * nbcon_emit_next_record - Emit a record in the acquired context
 * @wctxt:        The write context that will be handed to the write function
 * @use_atomic:        True if the write_atomic() callback is to be used
 *
 * Return:        True if this context still owns the console. False if
 *                ownership was handed over or taken.
 *
 * When this function returns false then the calling context no longer owns
 * the console and is no longer allowed to go forward. In this case it must
 * back out immediately and carefully. The buffer content is also no longer
 * trusted since it no longer belongs to the calling context. If the caller
 * wants to do more it must reacquire the console first.
 *
 * When true is returned, @wctxt->ctxt.backlog indicates whether there are
 * still records pending in the ringbuffer,
 */
static bool nbcon_emit_next_record(struct nbcon_write_context *wctxt, bool use_atomic)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        bool is_extended = console_srcu_read_flags(con) & CON_EXTENDED;
        struct printk_message pmsg = {
                .pbufs = ctxt->pbufs,
        };
        unsigned long con_dropped;
        struct nbcon_state cur;
        unsigned long dropped;
        unsigned long ulseq;

        /*
         * This function should never be called for consoles that have not
         * implemented the necessary callback for writing: i.e. legacy
         * consoles and, when atomic, nbcon consoles with no write_atomic().
         * Handle it as if ownership was lost and try to continue.
         *
         * Note that for nbcon consoles the write_thread() callback is
         * mandatory and was already checked in nbcon_alloc().
         */
        if (WARN_ON_ONCE((use_atomic && !con->write_atomic) ||
                         !(console_srcu_read_flags(con) & CON_NBCON))) {
                nbcon_context_release(ctxt);
                return false;
        }

        /*
         * The printk buffers are filled within an unsafe section. This
         * prevents NBCON_PRIO_NORMAL and NBCON_PRIO_EMERGENCY from
         * clobbering each other.
         */

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        ctxt->backlog = printk_get_next_message(&pmsg, ctxt->seq, is_extended, true);
        if (!ctxt->backlog)
                return nbcon_context_exit_unsafe(ctxt);

        /*
         * @con->dropped is not protected in case of an unsafe hostile
         * takeover. In that situation the update can be racy so
         * annotate it accordingly.
         */
        con_dropped = data_race(READ_ONCE(con->dropped));

        dropped = con_dropped + pmsg.dropped;
        if (dropped && !is_extended)
                console_prepend_dropped(&pmsg, dropped);

        /*
         * If the previous owner was assigned the same record, this context
         * has taken over ownership and is replaying the record. Prepend a
         * message to let the user know the record is replayed.
         */
        ulseq = atomic_long_read(&ACCESS_PRIVATE(con, nbcon_prev_seq));
        if (__ulseq_to_u64seq(prb, ulseq) == pmsg.seq) {
                console_prepend_replay(&pmsg);
        } else {
                /*
                 * Ensure this context is still the owner before trying to
                 * update @nbcon_prev_seq. Otherwise the value in @ulseq may
                 * not be from the previous owner and instead be some later
                 * value from the context that took over ownership.
                 */
                nbcon_state_read(con, &cur);
                if (!nbcon_context_can_proceed(ctxt, &cur))
                        return false;

                atomic_long_try_cmpxchg(&ACCESS_PRIVATE(con, nbcon_prev_seq), &ulseq,
                                        __u64seq_to_ulseq(pmsg.seq));
        }

        if (!nbcon_context_exit_unsafe(ctxt))
                return false;

        /* For skipped records just update seq/dropped in @con. */
        if (pmsg.outbuf_len == 0)
                goto update_con;

        /* Initialize the write context for driver callbacks. */
        nbcon_write_context_set_buf(wctxt, &pmsg.pbufs->outbuf[0], pmsg.outbuf_len);

        if (use_atomic)
                con->write_atomic(con, wctxt);
        else
                con->write_thread(con, wctxt);

        if (!wctxt->outbuf) {
                /*
                 * Ownership was lost and reacquired by the driver. Handle it
                 * as if ownership was lost.
                 */
                nbcon_context_release(ctxt);
                return false;
        }

        /*
         * Ownership may have been lost but _not_ reacquired by the driver.
         * This case is detected and handled when entering unsafe to update
         * dropped/seq values.
         */

        /*
         * Since any dropped message was successfully output, reset the
         * dropped count for the console.
         */
        dropped = 0;
update_con:
        /*
         * The dropped count and the sequence number are updated within an
         * unsafe section. This limits update races to the panic context and
         * allows the panic context to win.
         */

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        if (dropped != con_dropped) {
                /* Counterpart to the READ_ONCE() above. */
                WRITE_ONCE(con->dropped, dropped);
        }

        nbcon_seq_try_update(ctxt, pmsg.seq + 1);

        return nbcon_context_exit_unsafe(ctxt);
}

/*
 * nbcon_emit_one - Print one record for an nbcon console using the
 *                        specified callback
 * @wctxt:        An initialized write context struct to use for this context
 * @use_atomic:        True if the write_atomic() callback is to be used
 *
 * Return:        True, when a record has been printed and there are still
 *                pending records. The caller might want to continue flushing.
 *
 *                False, when there is no pending record, or when the console
 *                context cannot be acquired, or the ownership has been lost.
 *                The caller should give up. Either the job is done, cannot be
 *                done, or will be handled by the owning context.
 *
 * This is an internal helper to handle the locking of the console before
 * calling nbcon_emit_next_record().
 */
static bool nbcon_emit_one(struct nbcon_write_context *wctxt, bool use_atomic)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(wctxt, ctxt);
        struct console *con = ctxt->console;
        unsigned long flags;
        bool ret = false;

        if (!use_atomic) {
                con->device_lock(con, &flags);

                /*
                 * Ensure this stays on the CPU to make handover and
                 * takeover possible.
                 */
                cant_migrate();
        }

        if (!nbcon_context_try_acquire(ctxt))
                goto out;

        /*
         * nbcon_emit_next_record() returns false when the console was
         * handed over or taken over. In both cases the context is no
         * longer valid.
         *
         * The higher priority printing context takes over responsibility
         * to print the pending records.
         */
        if (!nbcon_emit_next_record(wctxt, use_atomic))
                goto out;

        nbcon_context_release(ctxt);

        ret = ctxt->backlog;
out:
        if (!use_atomic)
                con->device_unlock(con, flags);
        return ret;
}

/**
 * nbcon_kthread_should_wakeup - Check whether a printer thread should wakeup
 * @con:        Console to operate on
 * @ctxt:        The nbcon context from nbcon_context_try_acquire()
 *
 * Return:        True if the thread should shutdown or if the console is
 *                allowed to print and a record is available. False otherwise.
 *
 * After the thread wakes up, it must first check if it should shutdown before
 * attempting any printing.
 */
static bool nbcon_kthread_should_wakeup(struct console *con, struct nbcon_context *ctxt)
{
        bool ret = false;
        short flags;
        int cookie;

        if (kthread_should_stop())
                return true;

        cookie = console_srcu_read_lock();

        flags = console_srcu_read_flags(con);
        if (console_is_usable(con, flags, false)) {
                /* Bring the sequence in @ctxt up to date */
                ctxt->seq = nbcon_seq_read(con);

                ret = prb_read_valid(prb, ctxt->seq, NULL);
        }

        console_srcu_read_unlock(cookie);
        return ret;
}

/**
 * nbcon_kthread_func - The printer thread function
 * @__console:        Console to operate on
 *
 * Return:        0
 */
static int nbcon_kthread_func(void *__console)
{
        struct console *con = __console;
        struct nbcon_write_context wctxt = {
                .ctxt.console        = con,
                .ctxt.prio        = NBCON_PRIO_NORMAL,
        };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        short con_flags;
        bool backlog;
        int cookie;

wait_for_event:
        /*
         * Guarantee this task is visible on the rcuwait before
         * checking the wake condition.
         *
         * The full memory barrier within set_current_state() of
         * ___rcuwait_wait_event() pairs with the full memory
         * barrier within rcuwait_has_sleeper().
         *
         * This pairs with rcuwait_has_sleeper:A and nbcon_kthread_wake:A.
         */
        rcuwait_wait_event(&con->rcuwait,
                           nbcon_kthread_should_wakeup(con, ctxt),
                           TASK_INTERRUPTIBLE); /* LMM(nbcon_kthread_func:A) */

        do {
                if (kthread_should_stop())
                        return 0;

                backlog = false;

                /*
                 * Keep the srcu read lock around the entire operation so that
                 * synchronize_srcu() can guarantee that the kthread stopped
                 * or suspended printing.
                 */
                cookie = console_srcu_read_lock();

                con_flags = console_srcu_read_flags(con);

                if (console_is_usable(con, con_flags, false))
                        backlog = nbcon_emit_one(&wctxt, false);

                console_srcu_read_unlock(cookie);

                cond_resched();

        } while (backlog);

        goto wait_for_event;
}

/**
 * nbcon_irq_work - irq work to wake console printer thread
 * @irq_work:        The irq work to operate on
 */
static void nbcon_irq_work(struct irq_work *irq_work)
{
        struct console *con = container_of(irq_work, struct console, irq_work);

        nbcon_kthread_wake(con);
}

static inline bool rcuwait_has_sleeper(struct rcuwait *w)
{
        /*
         * Guarantee any new records can be seen by tasks preparing to wait
         * before this context checks if the rcuwait is empty.
         *
         * This full memory barrier pairs with the full memory barrier within
         * set_current_state() of ___rcuwait_wait_event(), which is called
         * after prepare_to_rcuwait() adds the waiter but before it has
         * checked the wait condition.
         *
         * This pairs with nbcon_kthread_func:A.
         */
        smp_mb(); /* LMM(rcuwait_has_sleeper:A) */
        return rcuwait_active(w);
}

/**
 * nbcon_kthreads_wake - Wake up printing threads using irq_work
 */
void nbcon_kthreads_wake(void)
{
        struct console *con;
        int cookie;

        if (!printk_kthreads_running)
                return;

        cookie = console_srcu_read_lock();
        for_each_console_srcu(con) {
                if (!(console_srcu_read_flags(con) & CON_NBCON))
                        continue;

                /*
                 * Only schedule irq_work if the printing thread is
                 * actively waiting. If not waiting, the thread will
                 * notice by itself that it has work to do.
                 */
                if (rcuwait_has_sleeper(&con->rcuwait))
                        irq_work_queue(&con->irq_work);
        }
        console_srcu_read_unlock(cookie);
}

/*
 * nbcon_kthread_stop - Stop a console printer thread
 * @con:        Console to operate on
 */
void nbcon_kthread_stop(struct console *con)
{
        lockdep_assert_console_list_lock_held();

        if (!con->kthread)
                return;

        kthread_stop(con->kthread);
        con->kthread = NULL;
}

/**
 * nbcon_kthread_create - Create a console printer thread
 * @con:        Console to operate on
 *
 * Return:        True if the kthread was started or already exists.
 *                Otherwise false and @con must not be registered.
 *
 * This function is called when it will be expected that nbcon consoles are
 * flushed using the kthread. The messages printed with NBCON_PRIO_NORMAL
 * will be no longer flushed by the legacy loop. This is why failure must
 * be fatal for console registration.
 *
 * If @con was already registered and this function fails, @con must be
 * unregistered before the global state variable @printk_kthreads_running
 * can be set.
 */
bool nbcon_kthread_create(struct console *con)
{
        struct task_struct *kt;

        lockdep_assert_console_list_lock_held();

        if (con->kthread)
                return true;

        kt = kthread_run(nbcon_kthread_func, con, "pr/%s%d", con->name, con->index);
        if (WARN_ON(IS_ERR(kt))) {
                con_printk(KERN_ERR, con, "failed to start printing thread\n");
                return false;
        }

        con->kthread = kt;

        /*
         * It is important that console printing threads are scheduled
         * shortly after a printk call and with generous runtime budgets.
         */
        sched_set_normal(con->kthread, -20);

        return true;
}

/* Track the nbcon emergency nesting per CPU. */
static DEFINE_PER_CPU(unsigned int, nbcon_pcpu_emergency_nesting);
static unsigned int early_nbcon_pcpu_emergency_nesting __initdata;

/**
 * nbcon_get_cpu_emergency_nesting - Get the per CPU emergency nesting pointer
 *
 * Context:        For reading, any context. For writing, any context which could
 *                not be migrated to another CPU.
 * Return:        Either a pointer to the per CPU emergency nesting counter of
 *                the current CPU or to the init data during early boot.
 *
 * The function is safe for reading per-CPU variables in any context because
 * preemption is disabled if the current CPU is in the emergency state. See
 * also nbcon_cpu_emergency_enter().
 */
static __ref unsigned int *nbcon_get_cpu_emergency_nesting(void)
{
        /*
         * The value of __printk_percpu_data_ready gets set in normal
         * context and before SMP initialization. As a result it could
         * never change while inside an nbcon emergency section.
         */
        if (!printk_percpu_data_ready())
                return &early_nbcon_pcpu_emergency_nesting;

        return raw_cpu_ptr(&nbcon_pcpu_emergency_nesting);
}

/**
 * nbcon_get_default_prio - The appropriate nbcon priority to use for nbcon
 *                                printing on the current CPU
 *
 * Context:        Any context.
 * Return:        The nbcon_prio to use for acquiring an nbcon console in this
 *                context for printing.
 *
 * The function is safe for reading per-CPU data in any context because
 * preemption is disabled if the current CPU is in the emergency or panic
 * state.
 */
enum nbcon_prio nbcon_get_default_prio(void)
{
        unsigned int *cpu_emergency_nesting;

        if (this_cpu_in_panic())
                return NBCON_PRIO_PANIC;

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
        if (*cpu_emergency_nesting)
                return NBCON_PRIO_EMERGENCY;

        return NBCON_PRIO_NORMAL;
}

/**
 * nbcon_legacy_emit_next_record - Print one record for an nbcon console
 *                                        in legacy contexts
 * @con:        The console to print on
 * @handover:        Will be set to true if a printk waiter has taken over the
 *                console_lock, in which case the caller is no longer holding
 *                both the console_lock and the SRCU read lock. Otherwise it
 *                is set to false.
 * @cookie:        The cookie from the SRCU read lock.
 * @use_atomic: Set true when called in an atomic or unknown context.
 *                It affects which nbcon callback will be used: write_atomic()
 *                or write_thread().
 *
 *                When false, the write_thread() callback is used and would be
 *                called in a preemtible context unless disabled by the
 *                device_lock. The legacy handover is not allowed in this mode.
 *
 * Context:        Any context except NMI.
 * Return:        True, when a record has been printed and there are still
 *                pending records. The caller might want to continue flushing.
 *
 *                False, when there is no pending record, or when the console
 *                context cannot be acquired, or the ownership has been lost.
 *                The caller should give up. Either the job is done, cannot be
 *                done, or will be handled by the owning context.
 *
 * This function is meant to be called by console_flush_all() to print records
 * on nbcon consoles from legacy context (printing via console unlocking).
 * Essentially it is the nbcon version of console_emit_next_record().
 */
bool nbcon_legacy_emit_next_record(struct console *con, bool *handover,
                                   int cookie, bool use_atomic)
{
        struct nbcon_write_context wctxt = { };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        unsigned long flags;
        bool progress;

        ctxt->console        = con;
        ctxt->prio        = nbcon_get_default_prio();

        if (use_atomic) {
                /*
                 * In an atomic or unknown context, use the same procedure as
                 * in console_emit_next_record(). It allows to handover.
                 */
                printk_safe_enter_irqsave(flags);
                console_lock_spinning_enable();
                stop_critical_timings();
        }

        progress = nbcon_emit_one(&wctxt, use_atomic);

        if (use_atomic) {
                start_critical_timings();
                *handover = console_lock_spinning_disable_and_check(cookie);
                printk_safe_exit_irqrestore(flags);
        } else {
                /* Non-atomic does not perform legacy spinning handovers. */
                *handover = false;
        }

        return progress;
}

/**
 * __nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
 *                                        write_atomic() callback
 * @con:                        The nbcon console to flush
 * @stop_seq:                        Flush up until this record
 * @allow_unsafe_takeover:        True, to allow unsafe hostile takeovers
 *
 * Return:        0 if @con was flushed up to @stop_seq Otherwise, error code on
 *                failure.
 *
 * Errors:
 *
 *        -EPERM:                Unable to acquire console ownership.
 *
 *        -EAGAIN:        Another context took over ownership while printing.
 *
 *        -ENOENT:        A record before @stop_seq is not available.
 *
 * If flushing up to @stop_seq was not successful, it only makes sense for the
 * caller to try again when -EAGAIN was returned. When -EPERM is returned,
 * this context is not allowed to acquire the console. When -ENOENT is
 * returned, it cannot be expected that the unfinalized record will become
 * available.
 */
static int __nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,
                                            bool allow_unsafe_takeover)
{
        struct nbcon_write_context wctxt = { };
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(&wctxt, ctxt);
        int err = 0;

        ctxt->console                        = con;
        ctxt->spinwait_max_us                = 2000;
        ctxt->prio                        = nbcon_get_default_prio();
        ctxt->allow_unsafe_takeover        = allow_unsafe_takeover;

        if (!nbcon_context_try_acquire(ctxt))
                return -EPERM;

        while (nbcon_seq_read(con) < stop_seq) {
                /*
                 * nbcon_emit_next_record() returns false when the console was
                 * handed over or taken over. In both cases the context is no
                 * longer valid.
                 */
                if (!nbcon_emit_next_record(&wctxt, true))
                        return -EAGAIN;

                if (!ctxt->backlog) {
                        /* Are there reserved but not yet finalized records? */
                        if (nbcon_seq_read(con) < stop_seq)
                                err = -ENOENT;
                        break;
                }
        }

        nbcon_context_release(ctxt);
        return err;
}

/**
 * nbcon_atomic_flush_pending_con - Flush specified nbcon console using its
 *                                        write_atomic() callback
 * @con:                        The nbcon console to flush
 * @stop_seq:                        Flush up until this record
 * @allow_unsafe_takeover:        True, to allow unsafe hostile takeovers
 *
 * This will stop flushing before @stop_seq if another context has ownership.
 * That context is then responsible for the flushing. Likewise, if new records
 * are added while this context was flushing and there is no other context
 * to handle the printing, this context must also flush those records.
 */
static void nbcon_atomic_flush_pending_con(struct console *con, u64 stop_seq,
                                           bool allow_unsafe_takeover)
{
        struct console_flush_type ft;
        unsigned long flags;
        int err;

again:
        /*
         * Atomic flushing does not use console driver synchronization (i.e.
         * it does not hold the port lock for uart consoles). Therefore IRQs
         * must be disabled to avoid being interrupted and then calling into
         * a driver that will deadlock trying to acquire console ownership.
         */
        local_irq_save(flags);

        err = __nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);

        local_irq_restore(flags);

        /*
         * If there was a new owner (-EPERM, -EAGAIN), that context is
         * responsible for completing.
         *
         * Do not wait for records not yet finalized (-ENOENT) to avoid a
         * possible deadlock. They will either get flushed by the writer or
         * eventually skipped on panic CPU.
         */
        if (err)
                return;

        /*
         * If flushing was successful but more records are available, this
         * context must flush those remaining records if the printer thread
         * is not available do it.
         */
        printk_get_console_flush_type(&ft);
        if (!ft.nbcon_offload &&
            prb_read_valid(prb, nbcon_seq_read(con), NULL)) {
                stop_seq = prb_next_reserve_seq(prb);
                goto again;
        }
}

/**
 * __nbcon_atomic_flush_pending - Flush all nbcon consoles using their
 *                                        write_atomic() callback
 * @stop_seq:                        Flush up until this record
 * @allow_unsafe_takeover:        True, to allow unsafe hostile takeovers
 */
static void __nbcon_atomic_flush_pending(u64 stop_seq, bool allow_unsafe_takeover)
{
        struct console *con;
        int cookie;

        cookie = console_srcu_read_lock();
        for_each_console_srcu(con) {
                short flags = console_srcu_read_flags(con);

                if (!(flags & CON_NBCON))
                        continue;

                if (!console_is_usable(con, flags, true))
                        continue;

                if (nbcon_seq_read(con) >= stop_seq)
                        continue;

                nbcon_atomic_flush_pending_con(con, stop_seq, allow_unsafe_takeover);
        }
        console_srcu_read_unlock(cookie);
}

/**
 * nbcon_atomic_flush_pending - Flush all nbcon consoles using their
 *                                write_atomic() callback
 *
 * Flush the backlog up through the currently newest record. Any new
 * records added while flushing will not be flushed if there is another
 * context available to handle the flushing. This is to avoid one CPU
 * printing unbounded because other CPUs continue to add records.
 */
void nbcon_atomic_flush_pending(void)
{
        __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), false);
}

/**
 * nbcon_atomic_flush_unsafe - Flush all nbcon consoles using their
 *        write_atomic() callback and allowing unsafe hostile takeovers
 *
 * Flush the backlog up through the currently newest record. Unsafe hostile
 * takeovers will be performed, if necessary.
 */
void nbcon_atomic_flush_unsafe(void)
{
        __nbcon_atomic_flush_pending(prb_next_reserve_seq(prb), true);
}

/**
 * nbcon_cpu_emergency_enter - Enter an emergency section where printk()
 *                                messages for that CPU are flushed directly
 *
 * Context:        Any context. Disables preemption.
 *
 * When within an emergency section, printk() calls will attempt to flush any
 * pending messages in the ringbuffer.
 */
void nbcon_cpu_emergency_enter(void)
{
        unsigned int *cpu_emergency_nesting;

        preempt_disable();

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();
        (*cpu_emergency_nesting)++;
}

/**
 * nbcon_cpu_emergency_exit - Exit an emergency section
 *
 * Context:        Within an emergency section. Enables preemption.
 */
void nbcon_cpu_emergency_exit(void)
{
        unsigned int *cpu_emergency_nesting;

        cpu_emergency_nesting = nbcon_get_cpu_emergency_nesting();

        if (!WARN_ON_ONCE(*cpu_emergency_nesting == 0))
                (*cpu_emergency_nesting)--;

        preempt_enable();
}

/**
 * nbcon_alloc - Allocate and init the nbcon console specific data
 * @con:        Console to initialize
 *
 * Return:        True if the console was fully allocated and initialized.
 *                Otherwise @con must not be registered.
 *
 * When allocation and init was successful, the console must be properly
 * freed using nbcon_free() once it is no longer needed.
 */
bool nbcon_alloc(struct console *con)
{
        struct nbcon_state state = { };

        /* The write_thread() callback is mandatory. */
        if (WARN_ON(!con->write_thread))
                return false;

        rcuwait_init(&con->rcuwait);
        init_irq_work(&con->irq_work, nbcon_irq_work);
        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL);
        nbcon_state_set(con, &state);

        /*
         * Initialize @nbcon_seq to the highest possible sequence number so
         * that practically speaking it will have nothing to print until a
         * desired initial sequence number has been set via nbcon_seq_force().
         */
        atomic_long_set(&ACCESS_PRIVATE(con, nbcon_seq), ULSEQ_MAX(prb));

        if (con->flags & CON_BOOT) {
                /*
                 * Boot console printing is synchronized with legacy console
                 * printing, so boot consoles can share the same global printk
                 * buffers.
                 */
                con->pbufs = &printk_shared_pbufs;
        } else {
                con->pbufs = kmalloc(sizeof(*con->pbufs), GFP_KERNEL);
                if (!con->pbufs) {
                        con_printk(KERN_ERR, con, "failed to allocate printing buffer\n");
                        return false;
                }

                if (printk_kthreads_running) {
                        if (!nbcon_kthread_create(con)) {
                                kfree(con->pbufs);
                                con->pbufs = NULL;
                                return false;
                        }
                }
        }

        return true;
}

/**
 * nbcon_free - Free and cleanup the nbcon console specific data
 * @con:        Console to free/cleanup nbcon data
 */
void nbcon_free(struct console *con)
{
        struct nbcon_state state = { };

        if (printk_kthreads_running)
                nbcon_kthread_stop(con);

        nbcon_state_set(con, &state);

        /* Boot consoles share global printk buffers. */
        if (!(con->flags & CON_BOOT))
                kfree(con->pbufs);

        con->pbufs = NULL;
}

/**
 * nbcon_device_try_acquire - Try to acquire nbcon console and enter unsafe
 *                                section
 * @con:        The nbcon console to acquire
 *
 * Context:        Under the locking mechanism implemented in
 *                @con->device_lock() including disabling migration.
 * Return:        True if the console was acquired. False otherwise.
 *
 * Console drivers will usually use their own internal synchronization
 * mechasism to synchronize between console printing and non-printing
 * activities (such as setting baud rates). However, nbcon console drivers
 * supporting atomic consoles may also want to mark unsafe sections when
 * performing non-printing activities in order to synchronize against their
 * atomic_write() callback.
 *
 * This function acquires the nbcon console using priority NBCON_PRIO_NORMAL
 * and marks it unsafe for handover/takeover.
 */
bool nbcon_device_try_acquire(struct console *con)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);

        cant_migrate();

        memset(ctxt, 0, sizeof(*ctxt));
        ctxt->console        = con;
        ctxt->prio        = NBCON_PRIO_NORMAL;

        if (!nbcon_context_try_acquire(ctxt))
                return false;

        if (!nbcon_context_enter_unsafe(ctxt))
                return false;

        return true;
}
EXPORT_SYMBOL_GPL(nbcon_device_try_acquire);

/**
 * nbcon_device_release - Exit unsafe section and release the nbcon console
 * @con:        The nbcon console acquired in nbcon_device_try_acquire()
 */
void nbcon_device_release(struct console *con)
{
        struct nbcon_context *ctxt = &ACCESS_PRIVATE(con, nbcon_device_ctxt);
        struct console_flush_type ft;
        int cookie;

        if (!nbcon_context_exit_unsafe(ctxt))
                return;

        nbcon_context_release(ctxt);

        /*
         * This context must flush any new records added while the console
         * was locked if the printer thread is not available to do it. The
         * console_srcu_read_lock must be taken to ensure the console is
         * usable throughout flushing.
         */
        cookie = console_srcu_read_lock();
        printk_get_console_flush_type(&ft);
        if (console_is_usable(con, console_srcu_read_flags(con), true) &&
            !ft.nbcon_offload &&
            prb_read_valid(prb, nbcon_seq_read(con), NULL)) {
                /*
                 * If nbcon_atomic flushing is not available, fallback to
                 * using the legacy loop.
                 */
                if (ft.nbcon_atomic) {
                        __nbcon_atomic_flush_pending_con(con, prb_next_reserve_seq(prb), false);
                } else if (ft.legacy_direct) {
                        if (console_trylock())
                                console_unlock();
                } else if (ft.legacy_offload) {
                        printk_trigger_flush();
                }
        }
        console_srcu_read_unlock(cookie);
}
EXPORT_SYMBOL_GPL(nbcon_device_release);

























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    6 


    6 











































































    4 





























































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/dsa/user.c - user device handling
 * Copyright (c) 2008-2009 Marvell Semiconductor
 */

#include <linux/list.h>
#include <linux/etherdevice.h>
#include <linux/netdevice.h>
#include <linux/phy.h>
#include <linux/phy_fixed.h>
#include <linux/phylink.h>
#include <linux/of_net.h>
#include <linux/of_mdio.h>
#include <linux/mdio.h>
#include <net/rtnetlink.h>
#include <net/pkt_cls.h>
#include <net/selftests.h>
#include <net/tc_act/tc_mirred.h>
#include <linux/if_bridge.h>
#include <linux/if_hsr.h>
#include <net/dcbnl.h>
#include <linux/netpoll.h>
#include <linux/string.h>

#include "conduit.h"
#include "dsa.h"
#include "netlink.h"
#include "port.h"
#include "switch.h"
#include "tag.h"
#include "user.h"

struct dsa_switchdev_event_work {
        struct net_device *dev;
        struct net_device *orig_dev;
        struct work_struct work;
        unsigned long event;
        /* Specific for SWITCHDEV_FDB_ADD_TO_DEVICE and
         * SWITCHDEV_FDB_DEL_TO_DEVICE
         */
        unsigned char addr[ETH_ALEN];
        u16 vid;
        bool host_addr;
};

enum dsa_standalone_event {
        DSA_UC_ADD,
        DSA_UC_DEL,
        DSA_MC_ADD,
        DSA_MC_DEL,
};

struct dsa_standalone_event_work {
        struct work_struct work;
        struct net_device *dev;
        enum dsa_standalone_event event;
        unsigned char addr[ETH_ALEN];
        u16 vid;
};

struct dsa_host_vlan_rx_filtering_ctx {
        struct net_device *dev;
        const unsigned char *addr;
        enum dsa_standalone_event event;
};

static bool dsa_switch_supports_uc_filtering(struct dsa_switch *ds)
{
        return ds->ops->port_fdb_add && ds->ops->port_fdb_del &&
               ds->fdb_isolation && !ds->vlan_filtering_is_global &&
               !ds->needs_standalone_vlan_filtering;
}

static bool dsa_switch_supports_mc_filtering(struct dsa_switch *ds)
{
        return ds->ops->port_mdb_add && ds->ops->port_mdb_del &&
               ds->fdb_isolation && !ds->vlan_filtering_is_global &&
               !ds->needs_standalone_vlan_filtering;
}

static void dsa_user_standalone_event_work(struct work_struct *work)
{
        struct dsa_standalone_event_work *standalone_work =
                container_of(work, struct dsa_standalone_event_work, work);
        const unsigned char *addr = standalone_work->addr;
        struct net_device *dev = standalone_work->dev;
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct switchdev_obj_port_mdb mdb;
        struct dsa_switch *ds = dp->ds;
        u16 vid = standalone_work->vid;
        int err;

        switch (standalone_work->event) {
        case DSA_UC_ADD:
                err = dsa_port_standalone_host_fdb_add(dp, addr, vid);
                if (err) {
                        dev_err(ds->dev,
                                "port %d failed to add %pM vid %d to fdb: %d\n",
                                dp->index, addr, vid, err);
                        break;
                }
                break;

        case DSA_UC_DEL:
                err = dsa_port_standalone_host_fdb_del(dp, addr, vid);
                if (err) {
                        dev_err(ds->dev,
                                "port %d failed to delete %pM vid %d from fdb: %d\n",
                                dp->index, addr, vid, err);
                }

                break;
        case DSA_MC_ADD:
                ether_addr_copy(mdb.addr, addr);
                mdb.vid = vid;

                err = dsa_port_standalone_host_mdb_add(dp, &mdb);
                if (err) {
                        dev_err(ds->dev,
                                "port %d failed to add %pM vid %d to mdb: %d\n",
                                dp->index, addr, vid, err);
                        break;
                }
                break;
        case DSA_MC_DEL:
                ether_addr_copy(mdb.addr, addr);
                mdb.vid = vid;

                err = dsa_port_standalone_host_mdb_del(dp, &mdb);
                if (err) {
                        dev_err(ds->dev,
                                "port %d failed to delete %pM vid %d from mdb: %d\n",
                                dp->index, addr, vid, err);
                }

                break;
        }

        kfree(standalone_work);
}

static int dsa_user_schedule_standalone_work(struct net_device *dev,
                                             enum dsa_standalone_event event,
                                             const unsigned char *addr,
                                             u16 vid)
{
        struct dsa_standalone_event_work *standalone_work;

        standalone_work = kzalloc(sizeof(*standalone_work), GFP_ATOMIC);
        if (!standalone_work)
                return -ENOMEM;

        INIT_WORK(&standalone_work->work, dsa_user_standalone_event_work);
        standalone_work->event = event;
        standalone_work->dev = dev;

        ether_addr_copy(standalone_work->addr, addr);
        standalone_work->vid = vid;

        dsa_schedule_work(&standalone_work->work);

        return 0;
}

static int dsa_user_host_vlan_rx_filtering(void *arg, int vid)
{
        struct dsa_host_vlan_rx_filtering_ctx *ctx = arg;

        return dsa_user_schedule_standalone_work(ctx->dev, ctx->event,
                                                  ctx->addr, vid);
}

static int dsa_user_vlan_for_each(struct net_device *dev,
                                  int (*cb)(void *arg, int vid), void *arg)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_vlan *v;
        int err;

        lockdep_assert_held(&dev->addr_list_lock);

        err = cb(arg, 0);
        if (err)
                return err;

        list_for_each_entry(v, &dp->user_vlans, list) {
                err = cb(arg, v->vid);
                if (err)
                        return err;
        }

        return 0;
}

static int dsa_user_sync_uc(struct net_device *dev,
                            const unsigned char *addr)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_host_vlan_rx_filtering_ctx ctx = {
                .dev = dev,
                .addr = addr,
                .event = DSA_UC_ADD,
        };

        dev_uc_add(conduit, addr);

        if (!dsa_switch_supports_uc_filtering(dp->ds))
                return 0;

        return dsa_user_vlan_for_each(dev, dsa_user_host_vlan_rx_filtering,
                                      &ctx);
}

static int dsa_user_unsync_uc(struct net_device *dev,
                              const unsigned char *addr)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_host_vlan_rx_filtering_ctx ctx = {
                .dev = dev,
                .addr = addr,
                .event = DSA_UC_DEL,
        };

        dev_uc_del(conduit, addr);

        if (!dsa_switch_supports_uc_filtering(dp->ds))
                return 0;

        return dsa_user_vlan_for_each(dev, dsa_user_host_vlan_rx_filtering,
                                      &ctx);
}

static int dsa_user_sync_mc(struct net_device *dev,
                            const unsigned char *addr)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_host_vlan_rx_filtering_ctx ctx = {
                .dev = dev,
                .addr = addr,
                .event = DSA_MC_ADD,
        };

        dev_mc_add(conduit, addr);

        if (!dsa_switch_supports_mc_filtering(dp->ds))
                return 0;

        return dsa_user_vlan_for_each(dev, dsa_user_host_vlan_rx_filtering,
                                      &ctx);
}

static int dsa_user_unsync_mc(struct net_device *dev,
                              const unsigned char *addr)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_host_vlan_rx_filtering_ctx ctx = {
                .dev = dev,
                .addr = addr,
                .event = DSA_MC_DEL,
        };

        dev_mc_del(conduit, addr);

        if (!dsa_switch_supports_mc_filtering(dp->ds))
                return 0;

        return dsa_user_vlan_for_each(dev, dsa_user_host_vlan_rx_filtering,
                                      &ctx);
}

void dsa_user_sync_ha(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        struct netdev_hw_addr *ha;

        netif_addr_lock_bh(dev);

        netdev_for_each_synced_mc_addr(ha, dev)
                dsa_user_sync_mc(dev, ha->addr);

        netdev_for_each_synced_uc_addr(ha, dev)
                dsa_user_sync_uc(dev, ha->addr);

        netif_addr_unlock_bh(dev);

        if (dsa_switch_supports_uc_filtering(ds) ||
            dsa_switch_supports_mc_filtering(ds))
                dsa_flush_workqueue();
}

void dsa_user_unsync_ha(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        struct netdev_hw_addr *ha;

        netif_addr_lock_bh(dev);

        netdev_for_each_synced_uc_addr(ha, dev)
                dsa_user_unsync_uc(dev, ha->addr);

        netdev_for_each_synced_mc_addr(ha, dev)
                dsa_user_unsync_mc(dev, ha->addr);

        netif_addr_unlock_bh(dev);

        if (dsa_switch_supports_uc_filtering(ds) ||
            dsa_switch_supports_mc_filtering(ds))
                dsa_flush_workqueue();
}

/* user mii_bus handling ***************************************************/
static int dsa_user_phy_read(struct mii_bus *bus, int addr, int reg)
{
        struct dsa_switch *ds = bus->priv;

        if (ds->phys_mii_mask & (1 << addr))
                return ds->ops->phy_read(ds, addr, reg);

        return 0xffff;
}

static int dsa_user_phy_write(struct mii_bus *bus, int addr, int reg, u16 val)
{
        struct dsa_switch *ds = bus->priv;

        if (ds->phys_mii_mask & (1 << addr))
                return ds->ops->phy_write(ds, addr, reg, val);

        return 0;
}

void dsa_user_mii_bus_init(struct dsa_switch *ds)
{
        ds->user_mii_bus->priv = (void *)ds;
        ds->user_mii_bus->name = "dsa user smi";
        ds->user_mii_bus->read = dsa_user_phy_read;
        ds->user_mii_bus->write = dsa_user_phy_write;
        snprintf(ds->user_mii_bus->id, MII_BUS_ID_SIZE, "dsa-%d.%d",
                 ds->dst->index, ds->index);
        ds->user_mii_bus->parent = ds->dev;
        ds->user_mii_bus->phy_mask = ~ds->phys_mii_mask;
}


/* user device handling ****************************************************/
static int dsa_user_get_iflink(const struct net_device *dev)
{
        return READ_ONCE(dsa_user_to_conduit(dev)->ifindex);
}

int dsa_user_host_uc_install(struct net_device *dev, const u8 *addr)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        int err;

        if (dsa_switch_supports_uc_filtering(ds)) {
                err = dsa_port_standalone_host_fdb_add(dp, addr, 0);
                if (err)
                        goto out;
        }

        if (!ether_addr_equal(addr, conduit->dev_addr)) {
                err = dev_uc_add(conduit, addr);
                if (err < 0)
                        goto del_host_addr;
        }

        return 0;

del_host_addr:
        if (dsa_switch_supports_uc_filtering(ds))
                dsa_port_standalone_host_fdb_del(dp, addr, 0);
out:
        return err;
}

void dsa_user_host_uc_uninstall(struct net_device *dev)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (!ether_addr_equal(dev->dev_addr, conduit->dev_addr))
                dev_uc_del(conduit, dev->dev_addr);

        if (dsa_switch_supports_uc_filtering(ds))
                dsa_port_standalone_host_fdb_del(dp, dev->dev_addr, 0);
}

static int dsa_user_open(struct net_device *dev)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        int err;

        err = dev_open(conduit, NULL);
        if (err < 0) {
                netdev_err(dev, "failed to open conduit %s\n", conduit->name);
                goto out;
        }

        err = dsa_user_host_uc_install(dev, dev->dev_addr);
        if (err)
                goto out;

        err = dsa_port_enable_rt(dp, dev->phydev);
        if (err)
                goto out_del_host_uc;

        return 0;

out_del_host_uc:
        dsa_user_host_uc_uninstall(dev);
out:
        return err;
}

static int dsa_user_close(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        dsa_port_disable_rt(dp);

        dsa_user_host_uc_uninstall(dev);

        return 0;
}

static void dsa_user_manage_host_flood(struct net_device *dev)
{
        bool mc = dev->flags & (IFF_PROMISC | IFF_ALLMULTI);
        struct dsa_port *dp = dsa_user_to_port(dev);
        bool uc = dev->flags & IFF_PROMISC;

        dsa_port_set_host_flood(dp, uc, mc);
}

static void dsa_user_change_rx_flags(struct net_device *dev, int change)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (change & IFF_ALLMULTI)
                dev_set_allmulti(conduit,
                                 dev->flags & IFF_ALLMULTI ? 1 : -1);
        if (change & IFF_PROMISC)
                dev_set_promiscuity(conduit,
                                    dev->flags & IFF_PROMISC ? 1 : -1);

        if (dsa_switch_supports_uc_filtering(ds) &&
            dsa_switch_supports_mc_filtering(ds))
                dsa_user_manage_host_flood(dev);
}

static void dsa_user_set_rx_mode(struct net_device *dev)
{
        __dev_mc_sync(dev, dsa_user_sync_mc, dsa_user_unsync_mc);
        __dev_uc_sync(dev, dsa_user_sync_uc, dsa_user_unsync_uc);
}

static int dsa_user_set_mac_address(struct net_device *dev, void *a)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        struct sockaddr *addr = a;
        int err;

        if (!is_valid_ether_addr(addr->sa_data))
                return -EADDRNOTAVAIL;

        if (ds->ops->port_set_mac_address) {
                err = ds->ops->port_set_mac_address(ds, dp->index,
                                                    addr->sa_data);
                if (err)
                        return err;
        }

        /* If the port is down, the address isn't synced yet to hardware or
         * to the DSA conduit, so there is nothing to change.
         */
        if (!(dev->flags & IFF_UP))
                goto out_change_dev_addr;

        err = dsa_user_host_uc_install(dev, addr->sa_data);
        if (err)
                return err;

        dsa_user_host_uc_uninstall(dev);

out_change_dev_addr:
        eth_hw_addr_set(dev, addr->sa_data);

        return 0;
}

struct dsa_user_dump_ctx {
        struct net_device *dev;
        struct sk_buff *skb;
        struct netlink_callback *cb;
        int idx;
};

static int
dsa_user_port_fdb_do_dump(const unsigned char *addr, u16 vid,
                          bool is_static, void *data)
{
        struct dsa_user_dump_ctx *dump = data;
        struct ndo_fdb_dump_context *ctx = (void *)dump->cb->ctx;
        u32 portid = NETLINK_CB(dump->cb->skb).portid;
        u32 seq = dump->cb->nlh->nlmsg_seq;
        struct nlmsghdr *nlh;
        struct ndmsg *ndm;

        if (dump->idx < ctx->fdb_idx)
                goto skip;

        nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH,
                        sizeof(*ndm), NLM_F_MULTI);
        if (!nlh)
                return -EMSGSIZE;

        ndm = nlmsg_data(nlh);
        ndm->ndm_family  = AF_BRIDGE;
        ndm->ndm_pad1    = 0;
        ndm->ndm_pad2    = 0;
        ndm->ndm_flags   = NTF_SELF;
        ndm->ndm_type    = 0;
        ndm->ndm_ifindex = dump->dev->ifindex;
        ndm->ndm_state   = is_static ? NUD_NOARP : NUD_REACHABLE;

        if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, addr))
                goto nla_put_failure;

        if (vid && nla_put_u16(dump->skb, NDA_VLAN, vid))
                goto nla_put_failure;

        nlmsg_end(dump->skb, nlh);

skip:
        dump->idx++;
        return 0;

nla_put_failure:
        nlmsg_cancel(dump->skb, nlh);
        return -EMSGSIZE;
}

static int
dsa_user_fdb_dump(struct sk_buff *skb, struct netlink_callback *cb,
                  struct net_device *dev, struct net_device *filter_dev,
                  int *idx)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_user_dump_ctx dump = {
                .dev = dev,
                .skb = skb,
                .cb = cb,
                .idx = *idx,
        };
        int err;

        err = dsa_port_fdb_dump(dp, dsa_user_port_fdb_do_dump, &dump);
        *idx = dump.idx;

        return err;
}

static int dsa_user_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
{
        struct dsa_user_priv *p = netdev_priv(dev);

        return phylink_mii_ioctl(p->dp->pl, ifr, cmd);
}

static int dsa_user_port_attr_set(struct net_device *dev, const void *ctx,
                                  const struct switchdev_attr *attr,
                                  struct netlink_ext_ack *extack)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        int ret;

        if (ctx && ctx != dp)
                return 0;

        switch (attr->id) {
        case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
                if (!dsa_port_offloads_bridge_port(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_set_state(dp, attr->u.stp_state, true);
                break;
        case SWITCHDEV_ATTR_ID_PORT_MST_STATE:
                if (!dsa_port_offloads_bridge_port(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_set_mst_state(dp, &attr->u.mst_state, extack);
                break;
        case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING:
                if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_vlan_filtering(dp, attr->u.vlan_filtering,
                                              extack);
                break;
        case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME:
                if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_ageing_time(dp, attr->u.ageing_time);
                break;
        case SWITCHDEV_ATTR_ID_BRIDGE_MST:
                if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_mst_enable(dp, attr->u.mst, extack);
                break;
        case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS:
                if (!dsa_port_offloads_bridge_port(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_pre_bridge_flags(dp, attr->u.brport_flags,
                                                extack);
                break;
        case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
                if (!dsa_port_offloads_bridge_port(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_bridge_flags(dp, attr->u.brport_flags, extack);
                break;
        case SWITCHDEV_ATTR_ID_VLAN_MSTI:
                if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev))
                        return -EOPNOTSUPP;

                ret = dsa_port_vlan_msti(dp, &attr->u.vlan_msti);
                break;
        default:
                ret = -EOPNOTSUPP;
                break;
        }

        return ret;
}

/* Must be called under rcu_read_lock() */
static int
dsa_user_vlan_check_for_8021q_uppers(struct net_device *user,
                                     const struct switchdev_obj_port_vlan *vlan)
{
        struct net_device *upper_dev;
        struct list_head *iter;

        netdev_for_each_upper_dev_rcu(user, upper_dev, iter) {
                u16 vid;

                if (!is_vlan_dev(upper_dev))
                        continue;

                vid = vlan_dev_vlan_id(upper_dev);
                if (vid == vlan->vid)
                        return -EBUSY;
        }

        return 0;
}

static int dsa_user_vlan_add(struct net_device *dev,
                             const struct switchdev_obj *obj,
                             struct netlink_ext_ack *extack)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct switchdev_obj_port_vlan *vlan;
        int err;

        if (dsa_port_skip_vlan_configuration(dp)) {
                NL_SET_ERR_MSG_MOD(extack, "skipping configuration of VLAN");
                return 0;
        }

        vlan = SWITCHDEV_OBJ_PORT_VLAN(obj);

        /* Deny adding a bridge VLAN when there is already an 802.1Q upper with
         * the same VID.
         */
        if (br_vlan_enabled(dsa_port_bridge_dev_get(dp))) {
                rcu_read_lock();
                err = dsa_user_vlan_check_for_8021q_uppers(dev, vlan);
                rcu_read_unlock();
                if (err) {
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Port already has a VLAN upper with this VID");
                        return err;
                }
        }

        return dsa_port_vlan_add(dp, vlan, extack);
}

/* Offload a VLAN installed on the bridge or on a foreign interface by
 * installing it as a VLAN towards the CPU port.
 */
static int dsa_user_host_vlan_add(struct net_device *dev,
                                  const struct switchdev_obj *obj,
                                  struct netlink_ext_ack *extack)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct switchdev_obj_port_vlan vlan;

        /* Do nothing if this is a software bridge */
        if (!dp->bridge)
                return -EOPNOTSUPP;

        if (dsa_port_skip_vlan_configuration(dp)) {
                NL_SET_ERR_MSG_MOD(extack, "skipping configuration of VLAN");
                return 0;
        }

        vlan = *SWITCHDEV_OBJ_PORT_VLAN(obj);

        /* Even though drivers often handle CPU membership in special ways,
         * it doesn't make sense to program a PVID, so clear this flag.
         */
        vlan.flags &= ~BRIDGE_VLAN_INFO_PVID;

        return dsa_port_host_vlan_add(dp, &vlan, extack);
}

static int dsa_user_port_obj_add(struct net_device *dev, const void *ctx,
                                 const struct switchdev_obj *obj,
                                 struct netlink_ext_ack *extack)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        int err;

        if (ctx && ctx != dp)
                return 0;

        switch (obj->id) {
        case SWITCHDEV_OBJ_ID_PORT_MDB:
                if (!dsa_port_offloads_bridge_port(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mdb_add(dp, SWITCHDEV_OBJ_PORT_MDB(obj));
                break;
        case SWITCHDEV_OBJ_ID_HOST_MDB:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_bridge_host_mdb_add(dp, SWITCHDEV_OBJ_PORT_MDB(obj));
                break;
        case SWITCHDEV_OBJ_ID_PORT_VLAN:
                if (dsa_port_offloads_bridge_port(dp, obj->orig_dev))
                        err = dsa_user_vlan_add(dev, obj, extack);
                else
                        err = dsa_user_host_vlan_add(dev, obj, extack);
                break;
        case SWITCHDEV_OBJ_ID_MRP:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mrp_add(dp, SWITCHDEV_OBJ_MRP(obj));
                break;
        case SWITCHDEV_OBJ_ID_RING_ROLE_MRP:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mrp_add_ring_role(dp,
                                                 SWITCHDEV_OBJ_RING_ROLE_MRP(obj));
                break;
        default:
                err = -EOPNOTSUPP;
                break;
        }

        return err;
}

static int dsa_user_vlan_del(struct net_device *dev,
                             const struct switchdev_obj *obj)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct switchdev_obj_port_vlan *vlan;

        if (dsa_port_skip_vlan_configuration(dp))
                return 0;

        vlan = SWITCHDEV_OBJ_PORT_VLAN(obj);

        return dsa_port_vlan_del(dp, vlan);
}

static int dsa_user_host_vlan_del(struct net_device *dev,
                                  const struct switchdev_obj *obj)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct switchdev_obj_port_vlan *vlan;

        /* Do nothing if this is a software bridge */
        if (!dp->bridge)
                return -EOPNOTSUPP;

        if (dsa_port_skip_vlan_configuration(dp))
                return 0;

        vlan = SWITCHDEV_OBJ_PORT_VLAN(obj);

        return dsa_port_host_vlan_del(dp, vlan);
}

static int dsa_user_port_obj_del(struct net_device *dev, const void *ctx,
                                 const struct switchdev_obj *obj)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        int err;

        if (ctx && ctx != dp)
                return 0;

        switch (obj->id) {
        case SWITCHDEV_OBJ_ID_PORT_MDB:
                if (!dsa_port_offloads_bridge_port(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mdb_del(dp, SWITCHDEV_OBJ_PORT_MDB(obj));
                break;
        case SWITCHDEV_OBJ_ID_HOST_MDB:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_bridge_host_mdb_del(dp, SWITCHDEV_OBJ_PORT_MDB(obj));
                break;
        case SWITCHDEV_OBJ_ID_PORT_VLAN:
                if (dsa_port_offloads_bridge_port(dp, obj->orig_dev))
                        err = dsa_user_vlan_del(dev, obj);
                else
                        err = dsa_user_host_vlan_del(dev, obj);
                break;
        case SWITCHDEV_OBJ_ID_MRP:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mrp_del(dp, SWITCHDEV_OBJ_MRP(obj));
                break;
        case SWITCHDEV_OBJ_ID_RING_ROLE_MRP:
                if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev))
                        return -EOPNOTSUPP;

                err = dsa_port_mrp_del_ring_role(dp,
                                                 SWITCHDEV_OBJ_RING_ROLE_MRP(obj));
                break;
        default:
                err = -EOPNOTSUPP;
                break;
        }

        return err;
}

static netdev_tx_t dsa_user_netpoll_send_skb(struct net_device *dev,
                                             struct sk_buff *skb)
{
#ifdef CONFIG_NET_POLL_CONTROLLER
        struct dsa_user_priv *p = netdev_priv(dev);

        return netpoll_send_skb(p->netpoll, skb);
#else
        BUG();
        return NETDEV_TX_OK;
#endif
}

static void dsa_skb_tx_timestamp(struct dsa_user_priv *p,
                                 struct sk_buff *skb)
{
        struct dsa_switch *ds = p->dp->ds;

        if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP_NOBPF))
                return;

        if (!ds->ops->port_txtstamp)
                return;

        ds->ops->port_txtstamp(ds, p->dp->index, skb);
}

netdev_tx_t dsa_enqueue_skb(struct sk_buff *skb, struct net_device *dev)
{
        /* SKB for netpoll still need to be mangled with the protocol-specific
         * tag to be successfully transmitted
         */
        if (unlikely(netpoll_tx_running(dev)))
                return dsa_user_netpoll_send_skb(dev, skb);

        /* Queue the SKB for transmission on the parent interface, but
         * do not modify its EtherType
         */
        skb->dev = dsa_user_to_conduit(dev);
        dev_queue_xmit(skb);

        return NETDEV_TX_OK;
}
EXPORT_SYMBOL_GPL(dsa_enqueue_skb);

static netdev_tx_t dsa_user_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct dsa_user_priv *p = netdev_priv(dev);
        struct sk_buff *nskb;

        dev_sw_netstats_tx_add(dev, 1, skb->len);

        memset(skb->cb, 0, sizeof(skb->cb));

        /* Handle tx timestamp if any */
        dsa_skb_tx_timestamp(p, skb);

        if (skb_ensure_writable_head_tail(skb, dev)) {
                dev_kfree_skb_any(skb);
                return NETDEV_TX_OK;
        }

        /* needed_tailroom should still be 'warm' in the cache line from
         * skb_ensure_writable_head_tail(), which has also ensured that
         * padding is safe.
         */
        if (dev->needed_tailroom)
                eth_skb_pad(skb);

        /* Transmit function may have to reallocate the original SKB,
         * in which case it must have freed it. Only free it here on error.
         */
        nskb = p->xmit(skb, dev);
        if (!nskb) {
                kfree_skb(skb);
                return NETDEV_TX_OK;
        }

        return dsa_enqueue_skb(nskb, dev);
}

/* ethtool operations *******************************************************/

static void dsa_user_get_drvinfo(struct net_device *dev,
                                 struct ethtool_drvinfo *drvinfo)
{
        strscpy(drvinfo->driver, "dsa", sizeof(drvinfo->driver));
        strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
        strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
}

static int dsa_user_get_regs_len(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_regs_len)
                return ds->ops->get_regs_len(ds, dp->index);

        return -EOPNOTSUPP;
}

static void
dsa_user_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *_p)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_regs)
                ds->ops->get_regs(ds, dp->index, regs, _p);
}

static int dsa_user_nway_reset(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        return phylink_ethtool_nway_reset(dp->pl);
}

static int dsa_user_get_eeprom_len(struct net_device *dev)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->cd && ds->cd->eeprom_len)
                return ds->cd->eeprom_len;

        if (ds->ops->get_eeprom_len)
                return ds->ops->get_eeprom_len(ds);

        return 0;
}

static int dsa_user_get_eeprom(struct net_device *dev,
                               struct ethtool_eeprom *eeprom, u8 *data)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_eeprom)
                return ds->ops->get_eeprom(ds, eeprom, data);

        return -EOPNOTSUPP;
}

static int dsa_user_set_eeprom(struct net_device *dev,
                               struct ethtool_eeprom *eeprom, u8 *data)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->set_eeprom)
                return ds->ops->set_eeprom(ds, eeprom, data);

        return -EOPNOTSUPP;
}

static void dsa_user_get_strings(struct net_device *dev,
                                 uint32_t stringset, uint8_t *data)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (stringset == ETH_SS_STATS) {
                ethtool_puts(&data, "tx_packets");
                ethtool_puts(&data, "tx_bytes");
                ethtool_puts(&data, "rx_packets");
                ethtool_puts(&data, "rx_bytes");
                if (ds->ops->get_strings)
                        ds->ops->get_strings(ds, dp->index, stringset, data);
        } else if (stringset ==  ETH_SS_TEST) {
                net_selftest_get_strings(data);
        }

}

static void dsa_user_get_ethtool_stats(struct net_device *dev,
                                       struct ethtool_stats *stats,
                                       uint64_t *data)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        struct pcpu_sw_netstats *s;
        unsigned int start;
        int i;

        for_each_possible_cpu(i) {
                u64 tx_packets, tx_bytes, rx_packets, rx_bytes;

                s = per_cpu_ptr(dev->tstats, i);
                do {
                        start = u64_stats_fetch_begin(&s->syncp);
                        tx_packets = u64_stats_read(&s->tx_packets);
                        tx_bytes = u64_stats_read(&s->tx_bytes);
                        rx_packets = u64_stats_read(&s->rx_packets);
                        rx_bytes = u64_stats_read(&s->rx_bytes);
                } while (u64_stats_fetch_retry(&s->syncp, start));
                data[0] += tx_packets;
                data[1] += tx_bytes;
                data[2] += rx_packets;
                data[3] += rx_bytes;
        }
        if (ds->ops->get_ethtool_stats)
                ds->ops->get_ethtool_stats(ds, dp->index, data + 4);
}

static int dsa_user_get_sset_count(struct net_device *dev, int sset)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (sset == ETH_SS_STATS) {
                int count = 0;

                if (ds->ops->get_sset_count) {
                        count = ds->ops->get_sset_count(ds, dp->index, sset);
                        if (count < 0)
                                return count;
                }

                return count + 4;
        } else if (sset ==  ETH_SS_TEST) {
                return net_selftest_get_count();
        }

        return -EOPNOTSUPP;
}

static void dsa_user_get_eth_phy_stats(struct net_device *dev,
                                       struct ethtool_eth_phy_stats *phy_stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_eth_phy_stats)
                ds->ops->get_eth_phy_stats(ds, dp->index, phy_stats);
}

static void dsa_user_get_eth_mac_stats(struct net_device *dev,
                                       struct ethtool_eth_mac_stats *mac_stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_eth_mac_stats)
                ds->ops->get_eth_mac_stats(ds, dp->index, mac_stats);
}

static void
dsa_user_get_eth_ctrl_stats(struct net_device *dev,
                            struct ethtool_eth_ctrl_stats *ctrl_stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_eth_ctrl_stats)
                ds->ops->get_eth_ctrl_stats(ds, dp->index, ctrl_stats);
}

static void
dsa_user_get_rmon_stats(struct net_device *dev,
                        struct ethtool_rmon_stats *rmon_stats,
                        const struct ethtool_rmon_hist_range **ranges)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_rmon_stats)
                ds->ops->get_rmon_stats(ds, dp->index, rmon_stats, ranges);
}

static void dsa_user_get_ts_stats(struct net_device *dev,
                                  struct ethtool_ts_stats *ts_stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_ts_stats)
                ds->ops->get_ts_stats(ds, dp->index, ts_stats);
}

static void dsa_user_net_selftest(struct net_device *ndev,
                                  struct ethtool_test *etest, u64 *buf)
{
        struct dsa_port *dp = dsa_user_to_port(ndev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->self_test) {
                ds->ops->self_test(ds, dp->index, etest, buf);
                return;
        }

        net_selftest(ndev, etest, buf);
}

static int dsa_user_get_mm(struct net_device *dev,
                           struct ethtool_mm_state *state)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (!ds->ops->get_mm)
                return -EOPNOTSUPP;

        return ds->ops->get_mm(ds, dp->index, state);
}

static int dsa_user_set_mm(struct net_device *dev, struct ethtool_mm_cfg *cfg,
                           struct netlink_ext_ack *extack)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (!ds->ops->set_mm)
                return -EOPNOTSUPP;

        return ds->ops->set_mm(ds, dp->index, cfg, extack);
}

static void dsa_user_get_mm_stats(struct net_device *dev,
                                  struct ethtool_mm_stats *stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_mm_stats)
                ds->ops->get_mm_stats(ds, dp->index, stats);
}

static void dsa_user_get_wol(struct net_device *dev, struct ethtool_wolinfo *w)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        phylink_ethtool_get_wol(dp->pl, w);

        if (ds->ops->get_wol)
                ds->ops->get_wol(ds, dp->index, w);
}

static int dsa_user_set_wol(struct net_device *dev, struct ethtool_wolinfo *w)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        int ret = -EOPNOTSUPP;

        phylink_ethtool_set_wol(dp->pl, w);

        if (ds->ops->set_wol)
                ret = ds->ops->set_wol(ds, dp->index, w);

        return ret;
}

static int dsa_user_set_eee(struct net_device *dev, struct ethtool_keee *e)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;
        int ret;

        /* Check whether the switch supports EEE */
        if (!ds->ops->support_eee || !ds->ops->support_eee(ds, dp->index))
                return -EOPNOTSUPP;

        /* If the port is using phylink managed EEE, then an unimplemented
         * set_mac_eee() is permissible.
         */
        if (!phylink_mac_implements_lpi(ds->phylink_mac_ops)) {
                /* Port's PHY and MAC both need to be EEE capable */
                if (!dev->phydev)
                        return -ENODEV;

                if (!ds->ops->set_mac_eee)
                        return -EOPNOTSUPP;

                ret = ds->ops->set_mac_eee(ds, dp->index, e);
                if (ret)
                        return ret;
        } else if (ds->ops->set_mac_eee) {
                ret = ds->ops->set_mac_eee(ds, dp->index, e);
                if (ret)
                        return ret;
        }

        return phylink_ethtool_set_eee(dp->pl, e);
}

static int dsa_user_get_eee(struct net_device *dev, struct ethtool_keee *e)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        /* Check whether the switch supports EEE */
        if (!ds->ops->support_eee || !ds->ops->support_eee(ds, dp->index))
                return -EOPNOTSUPP;

        /* Port's PHY and MAC both need to be EEE capable */
        if (!dev->phydev)
                return -ENODEV;

        return phylink_ethtool_get_eee(dp->pl, e);
}

static int dsa_user_get_link_ksettings(struct net_device *dev,
                                       struct ethtool_link_ksettings *cmd)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        return phylink_ethtool_ksettings_get(dp->pl, cmd);
}

static int dsa_user_set_link_ksettings(struct net_device *dev,
                                       const struct ethtool_link_ksettings *cmd)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        return phylink_ethtool_ksettings_set(dp->pl, cmd);
}

static void dsa_user_get_pause_stats(struct net_device *dev,
                                     struct ethtool_pause_stats *pause_stats)
{
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_switch *ds = dp->ds;

        if (ds->ops->get_pause_stats)
                ds->ops->get_pause_stats(ds, dp->index, pause_stats);
}

static void dsa_user_get_pauseparam(struct net_device *dev,
                                    struct ethtool_pauseparam *pause)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        phylink_ethtool_get_pauseparam(dp->pl, pause);
}

static int dsa_user_set_pauseparam(struct net_device *dev,
                                   struct ethtool_pauseparam *pause)
{
        struct dsa_port *dp = dsa_user_to_port(dev);

        return phylink_ethtool_set_pauseparam(dp->pl, pause);
}

#ifdef CONFIG_NET_POLL_CONTROLLER
static int dsa_user_netpoll_setup(struct net_device *dev)
{
        struct net_device *conduit = dsa_user_to_conduit(dev);
        struct dsa_user_priv *p = netdev_priv(dev);
        struct netpoll *netpoll;
        int err = 0;

        netpoll = kzalloc(sizeof(*netpoll), GFP_KERNEL);
        if (!netpoll)
                return -ENOMEM;

        err = __netpoll_setup(netpoll, conduit);
        if (err) {
                kfree(netpoll);
                goto out;
        }

        p->netpoll = netpoll;
out:
        return err;
}

static void dsa_user_netpoll_cleanup(struct net_device *dev)
{
        struct dsa_user_priv *p = netdev_priv(dev);
        struct netpoll *netpoll = p->netpoll;

        if (!netpoll)
                return;

        p->netpoll = NULL;

        __netpoll_free(netpoll);
}

static void dsa_user_poll_controller(struct net_device *dev)
{
}
#endif

static struct dsa_mall_tc_entry *
dsa_user_mall_tc_entry_find(struct net_device *dev, unsigned long cookie)
{
        struct dsa_user_priv *p = netdev_priv(dev);
        struct dsa_mall_tc_entry *mall_tc_entry;

        list_for_each_entry(mall_tc_entry, &p->mall_tc_list, list)
                if (mall_tc_entry->cookie == cookie)
                        return mall_tc_entry;

        return NULL;
}

static int
dsa_user_add_cls_matchall_mirred(struct net_device *dev,
                                 struct tc_cls_matchall_offload *cls,
                                 bool ingress, bool ingress_target)
{
        struct netlink_ext_ack *extack = cls->common.extack;
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_user_priv *p = netdev_priv(dev);
        struct dsa_mall_mirror_tc_entry *mirror;
        struct dsa_mall_tc_entry *mall_tc_entry;
        struct dsa_switch *ds = dp->ds;
        struct flow_action_entry *act;
        struct dsa_port *to_dp;
        int err;

        if (cls->common.protocol != htons(ETH_P_ALL)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Can only offload \"protocol all\" matchall filter");
                return -EOPNOTSUPP;
        }

        if (!ds->ops->port_mirror_add) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Switch does not support mirroring operation");
                return -EOPNOTSUPP;
        }

        if (!flow_action_basic_hw_stats_check(&cls->rule->action, extack))
                return -EOPNOTSUPP;

        act = &cls->rule->action.entries[0];

        if (!act->dev)
                return -EINVAL;

        if (dsa_user_dev_check(act->dev)) {
                if (ingress_target) {
                        /* We can only fulfill this using software assist */
                        if (cls->common.skip_sw) {
                                NL_SET_ERR_MSG_MOD(extack,
                                                   "Can only mirred to ingress of DSA user port if filter also runs in software");
                                return -EOPNOTSUPP;
                        }
                        to_dp = dp->cpu_dp;
                } else {
                        to_dp = dsa_user_to_port(act->dev);
                }
        } else {
                /* Handle mirroring to foreign target ports as a mirror towards
                 * the CPU. The software tc rule will take the packets from
                 * there.
                 */
                if (cls->common.skip_sw) {
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Can only mirred to CPU if filter also runs in software");
                        return -EOPNOTSUPP;
                }
                to_dp = dp->cpu_dp;
        }

        if (dp->ds != to_dp->ds) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Cross-chip mirroring not implemented");
                return -EOPNOTSUPP;
        }

        mall_tc_entry = kzalloc(sizeof(*mall_tc_entry), GFP_KERNEL);
        if (!mall_tc_entry)
                return -ENOMEM;

        mall_tc_entry->cookie = cls->cookie;
        mall_tc_entry->type = DSA_PORT_MALL_MIRROR;
        mirror = &mall_tc_entry->mirror;
        mirror->to_local_port = to_dp->index;
        mirror->ingress = ingress;

        err = ds->ops->port_mirror_add(ds, dp->index, mirror, ingress, extack);
        if (err) {
                kfree(mall_tc_entry);
                return err;
        }

        list_add_tail(&mall_tc_entry->list, &p->mall_tc_list);

        return err;
}

static int
dsa_user_add_cls_matchall_police(struct net_device *dev,
                                 struct tc_cls_matchall_offload *cls,
                                 bool ingress)
{
        struct netlink_ext_ack *extack = cls->common.extack;
        struct dsa_port *dp = dsa_user_to_port(dev);
        struct dsa_user_priv *p = netdev_priv(dev);
        struct dsa_mall_policer_tc_entry *policer;
        struct dsa_mall_tc_entry *mall_tc_entry;
        struct dsa_switch *ds = dp->ds;
        struct flow_action_entry *act;
        int err;

        if (!ds->ops->port_policer_add) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Policing offload not implemented");
                return -EOPNOTSUPP;
        }

        if (!ingress) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Only supported on ingress qdisc");
                return -EOPNOTSUPP;
        }

        if (!flow_action_basic_hw_stats_check(&cls->rule->action, extack))
                return -EOPNOTSUPP;

        list_for_each_entry(mall_tc_entry, &p->mall_tc_list, list) {
                if (mall_tc_entry->type == DSA_PORT_MALL_POLICER) {
                        NL_SET_ERR_MSG_MOD(extack,
                                           "Only one port policer allowed");
                        return -EEXIST;
                }
        }

        act = &cls->rule->action.entries[0];

        mall_tc_entry = kzalloc(sizeof(*mall_tc_entry), GFP_KERNEL);
        if (!mall_tc_entry)
                return -ENOMEM;

        mall_tc_entry->cookie = cls->cookie;
        mall_tc_entry->type = DSA_PORT_MALL_POLICER;
        policer = &mall_tc_entry->policer;
        policer->rate_bytes_per_sec = act->police.rate_bytes_ps;
        policer->burst = act->police.burst;

        err = ds->ops->port_policer_add(ds, dp->index, policer);
        if (err) {
                kfree(mall_tc_entry);
                return err;
        }

        list_add_tail(&mall_tc_entry->list, &p->mall_tc_list);

        return err;
}

static int dsa_user_add_cls_matchall(struct net_device *dev,
                                     struct tc_cls_matchall_offload *cls,
                                     bool ingress)
{
        const struct flow_action *action = &cls->rule->action;
        struct netlink_ext_ack *extack = cls->common.extack;

        if (!flow_offload_has_one_action(action)) {
                NL_SET_ERR_MSG_MOD(extack,
                                   "Cannot offload matchall filter with more than one action");