Total coverage: 284635 (16%)of 1874298 





















































































































































  218 
  232 






























































   41 


    3 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
/*
 * net/tipc/core.h: Include file for TIPC global declarations
 *
 * Copyright (c) 2005-2006, 2013-2018 Ericsson AB
 * Copyright (c) 2005-2007, 2010-2013, Wind River Systems
 * Copyright (c) 2020, Red Hat Inc
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the names of the copyright holders nor the names of its
 *    contributors may be used to endorse or promote products derived from
 *    this software without specific prior written permission.
 *
 * Alternatively, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") version 2 as published by the Free
 * Software Foundation.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _TIPC_CORE_H
#define _TIPC_CORE_H

#include <linux/tipc.h>
#include <linux/tipc_config.h>
#include <linux/tipc_netlink.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/timer.h>
#include <linux/string.h>
#include <linux/uaccess.h>
#include <linux/interrupt.h>
#include <linux/atomic.h>
#include <linux/netdevice.h>
#include <linux/in.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/rtnetlink.h>
#include <linux/etherdevice.h>
#include <net/netns/generic.h>
#include <linux/rhashtable.h>
#include <net/genetlink.h>
#include <net/netns/hash.h>

#ifdef pr_fmt
#undef pr_fmt
#endif

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

struct tipc_node;
struct tipc_bearer;
struct tipc_bc_base;
struct tipc_link;
struct tipc_topsrv;
struct tipc_monitor;
#ifdef CONFIG_TIPC_CRYPTO
struct tipc_crypto;
#endif

#define TIPC_MOD_VER "2.0.0"

#define NODE_HTABLE_SIZE       512
#define MAX_BEARERS                 3
#define TIPC_DEF_MON_THRESHOLD  32
#define NODE_ID_LEN             16
#define NODE_ID_STR_LEN        (NODE_ID_LEN * 2 + 1)

extern unsigned int tipc_net_id __read_mostly;
extern int sysctl_tipc_rmem[3] __read_mostly;
extern int sysctl_tipc_named_timeout __read_mostly;

struct tipc_net {
        u8  node_id[NODE_ID_LEN];
        u32 node_addr;
        u32 trial_addr;
        unsigned long addr_trial_end;
        char node_id_string[NODE_ID_STR_LEN];
        int net_id;
        int random;
        bool legacy_addr_format;

        /* Node table and node list */
        spinlock_t node_list_lock;
        struct hlist_head node_htable[NODE_HTABLE_SIZE];
        struct list_head node_list;
        u32 num_nodes;
        u32 num_links;

        /* Neighbor monitoring list */
        struct tipc_monitor *monitors[MAX_BEARERS];
        int mon_threshold;

        /* Bearer list */
        struct tipc_bearer __rcu *bearer_list[MAX_BEARERS + 1];

        /* Broadcast link */
        spinlock_t bclock;
        struct tipc_bc_base *bcbase;
        struct tipc_link *bcl;

        /* Socket hash table */
        struct rhashtable sk_rht;

        /* Name table */
        spinlock_t nametbl_lock;
        struct name_table *nametbl;

        /* Topology subscription server */
        struct tipc_topsrv *topsrv;
        atomic_t subscription_count;

        /* Cluster capabilities */
        u16 capabilities;

        /* Tracing of node internal messages */
        struct packet_type loopback_pt;

#ifdef CONFIG_TIPC_CRYPTO
        /* TX crypto handler */
        struct tipc_crypto *crypto_tx;
#endif
        /* Work item for net finalize */
        struct work_struct work;
        /* The numbers of work queues in schedule */
        atomic_t wq_count;
};

static inline struct tipc_net *tipc_net(struct net *net)
{
        return net_generic(net, tipc_net_id);
}

static inline int tipc_netid(struct net *net)
{
        return tipc_net(net)->net_id;
}

static inline struct list_head *tipc_nodes(struct net *net)
{
        return &tipc_net(net)->node_list;
}

static inline struct name_table *tipc_name_table(struct net *net)
{
        return tipc_net(net)->nametbl;
}

static inline struct tipc_topsrv *tipc_topsrv(struct net *net)
{
        return tipc_net(net)->topsrv;
}

static inline unsigned int tipc_hashfn(u32 addr)
{
        return addr & (NODE_HTABLE_SIZE - 1);
}

static inline u16 mod(u16 x)
{
        return x & 0xffffu;
}

static inline int less_eq(u16 left, u16 right)
{
        return mod(right - left) < 32768u;
}

static inline int more(u16 left, u16 right)
{
        return !less_eq(left, right);
}

static inline int less(u16 left, u16 right)
{
        return less_eq(left, right) && (mod(right) != mod(left));
}

static inline int tipc_in_range(u16 val, u16 min, u16 max)
{
        return !less(val, min) && !more(val, max);
}

static inline u32 tipc_net_hash_mixes(struct net *net, int tn_rand)
{
        return net_hash_mix(&init_net) ^ net_hash_mix(net) ^ tn_rand;
}

static inline u32 hash128to32(char *bytes)
{
        __be32 *tmp = (__be32 *)bytes;
        u32 res;

        res = ntohl(tmp[0] ^ tmp[1] ^ tmp[2] ^ tmp[3]);
        if (likely(res))
                return res;
        return  ntohl(tmp[0] | tmp[1] | tmp[2] | tmp[3]);
}

#ifdef CONFIG_SYSCTL
int tipc_register_sysctl(void);
void tipc_unregister_sysctl(void);
#else
#define tipc_register_sysctl() 0
#define tipc_unregister_sysctl()
#endif
#endif

































































































    7 









  227 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
// SPDX-License-Identifier: GPL-2.0
/* User-mappable watch queue
 *
 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * See Documentation/core-api/watch_queue.rst
 */

#ifndef _LINUX_WATCH_QUEUE_H
#define _LINUX_WATCH_QUEUE_H

#include <uapi/linux/watch_queue.h>
#include <linux/kref.h>
#include <linux/rcupdate.h>

#ifdef CONFIG_WATCH_QUEUE

struct cred;

struct watch_type_filter {
        enum watch_notification_type type;
        __u32                subtype_filter[1];        /* Bitmask of subtypes to filter on */
        __u32                info_filter;                /* Filter on watch_notification::info */
        __u32                info_mask;                /* Mask of relevant bits in info_filter */
};

struct watch_filter {
        union {
                struct rcu_head        rcu;
                /* Bitmask of accepted types */
                DECLARE_BITMAP(type_filter, WATCH_TYPE__NR);
        };
        u32                        nr_filters;        /* Number of filters */
        struct watch_type_filter filters[] __counted_by(nr_filters);
};

struct watch_queue {
        struct rcu_head                rcu;
        struct watch_filter __rcu *filter;
        struct pipe_inode_info        *pipe;                /* Pipe we use as a buffer, NULL if queue closed */
        struct hlist_head        watches;        /* Contributory watches */
        struct page                **notes;        /* Preallocated notifications */
        unsigned long                *notes_bitmap;        /* Allocation bitmap for notes */
        struct kref                usage;                /* Object usage count */
        spinlock_t                lock;
        unsigned int                nr_notes;        /* Number of notes */
        unsigned int                nr_pages;        /* Number of pages in notes[] */
};

/*
 * Representation of a watch on an object.
 */
struct watch {
        union {
                struct rcu_head        rcu;
                u32                info_id;        /* ID to be OR'd in to info field */
        };
        struct watch_queue __rcu *queue;        /* Queue to post events to */
        struct hlist_node        queue_node;        /* Link in queue->watches */
        struct watch_list __rcu        *watch_list;
        struct hlist_node        list_node;        /* Link in watch_list->watchers */
        const struct cred        *cred;                /* Creds of the owner of the watch */
        void                        *private;        /* Private data for the watched object */
        u64                        id;                /* Internal identifier */
        struct kref                usage;                /* Object usage count */
};

/*
 * List of watches on an object.
 */
struct watch_list {
        struct rcu_head                rcu;
        struct hlist_head        watchers;
        void (*release_watch)(struct watch *);
        spinlock_t                lock;
};

extern void __post_watch_notification(struct watch_list *,
                                      struct watch_notification *,
                                      const struct cred *,
                                      u64);
extern struct watch_queue *get_watch_queue(int);
extern void put_watch_queue(struct watch_queue *);
extern void init_watch(struct watch *, struct watch_queue *);
extern int add_watch_to_object(struct watch *, struct watch_list *);
extern int remove_watch_from_object(struct watch_list *, struct watch_queue *, u64, bool);
extern long watch_queue_set_size(struct pipe_inode_info *, unsigned int);
extern long watch_queue_set_filter(struct pipe_inode_info *,
                                   struct watch_notification_filter __user *);
extern int watch_queue_init(struct pipe_inode_info *);
extern void watch_queue_clear(struct watch_queue *);

static inline void init_watch_list(struct watch_list *wlist,
                                   void (*release_watch)(struct watch *))
{
        INIT_HLIST_HEAD(&wlist->watchers);
        spin_lock_init(&wlist->lock);
        wlist->release_watch = release_watch;
}

static inline void post_watch_notification(struct watch_list *wlist,
                                           struct watch_notification *n,
                                           const struct cred *cred,
                                           u64 id)
{
        if (unlikely(wlist))
                __post_watch_notification(wlist, n, cred, id);
}

static inline void remove_watch_list(struct watch_list *wlist, u64 id)
{
        if (wlist) {
                remove_watch_from_object(wlist, NULL, id, true);
                kfree_rcu(wlist, rcu);
        }
}

/**
 * watch_sizeof - Calculate the information part of the size of a watch record,
 * given the structure size.
 */
#define watch_sizeof(STRUCT) (sizeof(STRUCT) << WATCH_INFO_LENGTH__SHIFT)

#else
static inline int watch_queue_init(struct pipe_inode_info *pipe)
{
        return -ENOPKG;
}

#endif

#endif /* _LINUX_WATCH_QUEUE_H */



































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
// SPDX-License-Identifier: GPL-2.0
/*
 * HugeTLB Vmemmap Optimization (HVO)
 *
 * Copyright (c) 2020, ByteDance. All rights reserved.
 *
 *     Author: Muchun Song <songmuchun@bytedance.com>
 */
#ifndef _LINUX_HUGETLB_VMEMMAP_H
#define _LINUX_HUGETLB_VMEMMAP_H
#include <linux/hugetlb.h>
#include <linux/io.h>
#include <linux/memblock.h>

/*
 * Reserve one vmemmap page, all vmemmap addresses are mapped to it. See
 * Documentation/mm/vmemmap_dedup.rst.
 */
#define HUGETLB_VMEMMAP_RESERVE_SIZE        PAGE_SIZE
#define HUGETLB_VMEMMAP_RESERVE_PAGES        (HUGETLB_VMEMMAP_RESERVE_SIZE / sizeof(struct page))

#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio);
long hugetlb_vmemmap_restore_folios(const struct hstate *h,
                                        struct list_head *folio_list,
                                        struct list_head *non_hvo_folios);
void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio);
void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);
void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h, struct list_head *folio_list);
#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT
void hugetlb_vmemmap_init_early(int nid);
void hugetlb_vmemmap_init_late(int nid);
#endif


static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h)
{
        return pages_per_huge_page(h) * sizeof(struct page);
}

/*
 * Return how many vmemmap size associated with a HugeTLB page that can be
 * optimized and can be freed to the buddy allocator.
 */
static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
        int size = hugetlb_vmemmap_size(h) - HUGETLB_VMEMMAP_RESERVE_SIZE;

        if (!is_power_of_2(sizeof(struct page)))
                return 0;
        return size > 0 ? size : 0;
}
#else
static inline int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio)
{
        return 0;
}

static inline long hugetlb_vmemmap_restore_folios(const struct hstate *h,
                                        struct list_head *folio_list,
                                        struct list_head *non_hvo_folios)
{
        list_splice_init(folio_list, non_hvo_folios);
        return 0;
}

static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio)
{
}

static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list)
{
}

static inline void hugetlb_vmemmap_optimize_bootmem_folios(struct hstate *h,
                                                struct list_head *folio_list)
{
}

static inline void hugetlb_vmemmap_init_early(int nid)
{
}

static inline void hugetlb_vmemmap_init_late(int nid)
{
}

static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate *h)
{
        return 0;
}
#endif /* CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP */

static inline bool hugetlb_vmemmap_optimizable(const struct hstate *h)
{
        return hugetlb_vmemmap_optimizable_size(h) != 0;
}
#endif /* _LINUX_HUGETLB_VMEMMAP_H */





































   24 
















  114 
  128 













































































  323 



   45 
   45 

































































































    1 
    1 

    1 











   15 


 10478 





    4 
  112 
























































   10 


 6366 










   32 






    9 
















    2 












    3 
















   30 












    1 








































    1 



























    4 




























 4917 











































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_MM_H
#define _LINUX_SCHED_MM_H

#include <linux/kernel.h>
#include <linux/atomic.h>
#include <linux/sched.h>
#include <linux/mm_types.h>
#include <linux/gfp.h>
#include <linux/sync_core.h>
#include <linux/sched/coredump.h>

/*
 * Routines for handling mm_structs
 */
extern struct mm_struct *mm_alloc(void);

/**
 * mmgrab() - Pin a &struct mm_struct.
 * @mm: The &struct mm_struct to pin.
 *
 * Make sure that @mm will not get freed even after the owning task
 * exits. This doesn't guarantee that the associated address space
 * will still exist later on and mmget_not_zero() has to be used before
 * accessing it.
 *
 * This is a preferred way to pin @mm for a longer/unbounded amount
 * of time.
 *
 * Use mmdrop() to release the reference acquired by mmgrab().
 *
 * See also <Documentation/mm/active_mm.rst> for an in-depth explanation
 * of &mm_struct.mm_count vs &mm_struct.mm_users.
 */
static inline void mmgrab(struct mm_struct *mm)
{
        atomic_inc(&mm->mm_count);
}

static inline void smp_mb__after_mmgrab(void)
{
        smp_mb__after_atomic();
}

extern void __mmdrop(struct mm_struct *mm);

static inline void mmdrop(struct mm_struct *mm)
{
        /*
         * The implicit full barrier implied by atomic_dec_and_test() is
         * required by the membarrier system call before returning to
         * user-space, after storing to rq->curr.
         */
        if (unlikely(atomic_dec_and_test(&mm->mm_count)))
                __mmdrop(mm);
}

#ifdef CONFIG_PREEMPT_RT
/*
 * RCU callback for delayed mm drop. Not strictly RCU, but call_rcu() is
 * by far the least expensive way to do that.
 */
static inline void __mmdrop_delayed(struct rcu_head *rhp)
{
        struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop);

        __mmdrop(mm);
}

/*
 * Invoked from finish_task_switch(). Delegates the heavy lifting on RT
 * kernels via RCU.
 */
static inline void mmdrop_sched(struct mm_struct *mm)
{
        /* Provides a full memory barrier. See mmdrop() */
        if (atomic_dec_and_test(&mm->mm_count))
                call_rcu(&mm->delayed_drop, __mmdrop_delayed);
}
#else
static inline void mmdrop_sched(struct mm_struct *mm)
{
        mmdrop(mm);
}
#endif

/* Helpers for lazy TLB mm refcounting */
static inline void mmgrab_lazy_tlb(struct mm_struct *mm)
{
        if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT))
                mmgrab(mm);
}

static inline void mmdrop_lazy_tlb(struct mm_struct *mm)
{
        if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) {
                mmdrop(mm);
        } else {
                /*
                 * mmdrop_lazy_tlb must provide a full memory barrier, see the
                 * membarrier comment finish_task_switch which relies on this.
                 */
                smp_mb();
        }
}

static inline void mmdrop_lazy_tlb_sched(struct mm_struct *mm)
{
        if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT))
                mmdrop_sched(mm);
        else
                smp_mb(); /* see mmdrop_lazy_tlb() above */
}

/**
 * mmget() - Pin the address space associated with a &struct mm_struct.
 * @mm: The address space to pin.
 *
 * Make sure that the address space of the given &struct mm_struct doesn't
 * go away. This does not protect against parts of the address space being
 * modified or freed, however.
 *
 * Never use this function to pin this address space for an
 * unbounded/indefinite amount of time.
 *
 * Use mmput() to release the reference acquired by mmget().
 *
 * See also <Documentation/mm/active_mm.rst> for an in-depth explanation
 * of &mm_struct.mm_count vs &mm_struct.mm_users.
 */
static inline void mmget(struct mm_struct *mm)
{
        atomic_inc(&mm->mm_users);
}

static inline bool mmget_not_zero(struct mm_struct *mm)
{
        return atomic_inc_not_zero(&mm->mm_users);
}

/* mmput gets rid of the mappings and all user-space */
extern void mmput(struct mm_struct *);
#ifdef CONFIG_MMU
/* same as above but performs the slow path from the async context. Can
 * be called from the atomic context as well
 */
void mmput_async(struct mm_struct *);
#endif

/* Grab a reference to a task's mm, if it is not already going away */
extern struct mm_struct *get_task_mm(struct task_struct *task);
/*
 * Grab a reference to a task's mm, if it is not already going away
 * and ptrace_may_access with the mode parameter passed to it
 * succeeds.
 */
extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
/* Remove the current tasks stale references to the old mm_struct on exit() */
extern void exit_mm_release(struct task_struct *, struct mm_struct *);
/* Remove the current tasks stale references to the old mm_struct on exec() */
extern void exec_mm_release(struct task_struct *, struct mm_struct *);

#ifdef CONFIG_MEMCG
extern void mm_update_next_owner(struct mm_struct *mm);
#else
static inline void mm_update_next_owner(struct mm_struct *mm)
{
}
#endif /* CONFIG_MEMCG */

#ifdef CONFIG_MMU
#ifndef arch_get_mmap_end
#define arch_get_mmap_end(addr, len, flags)        (TASK_SIZE)
#endif

#ifndef arch_get_mmap_base
#define arch_get_mmap_base(addr, base) (base)
#endif

extern void arch_pick_mmap_layout(struct mm_struct *mm,
                                  struct rlimit *rlim_stack);

unsigned long
arch_get_unmapped_area(struct file *filp, unsigned long addr,
                       unsigned long len, unsigned long pgoff,
                       unsigned long flags, vm_flags_t vm_flags);
unsigned long
arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
                               unsigned long len, unsigned long pgoff,
                               unsigned long flags, vm_flags_t);

unsigned long mm_get_unmapped_area(struct mm_struct *mm, struct file *filp,
                                   unsigned long addr, unsigned long len,
                                   unsigned long pgoff, unsigned long flags);

unsigned long mm_get_unmapped_area_vmflags(struct mm_struct *mm,
                                           struct file *filp,
                                           unsigned long addr,
                                           unsigned long len,
                                           unsigned long pgoff,
                                           unsigned long flags,
                                           vm_flags_t vm_flags);

unsigned long
generic_get_unmapped_area(struct file *filp, unsigned long addr,
                          unsigned long len, unsigned long pgoff,
                          unsigned long flags, vm_flags_t vm_flags);
unsigned long
generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
                                  unsigned long len, unsigned long pgoff,
                                  unsigned long flags, vm_flags_t vm_flags);
#else
static inline void arch_pick_mmap_layout(struct mm_struct *mm,
                                         struct rlimit *rlim_stack) {}
#endif

static inline bool in_vfork(struct task_struct *tsk)
{
        bool ret;

        /*
         * need RCU to access ->real_parent if CLONE_VM was used along with
         * CLONE_PARENT.
         *
         * We check real_parent->mm == tsk->mm because CLONE_VFORK does not
         * imply CLONE_VM
         *
         * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
         * ->real_parent is not necessarily the task doing vfork(), so in
         * theory we can't rely on task_lock() if we want to dereference it.
         *
         * And in this case we can't trust the real_parent->mm == tsk->mm
         * check, it can be false negative. But we do not care, if init or
         * another oom-unkillable task does this it should blame itself.
         */
        rcu_read_lock();
        ret = tsk->vfork_done &&
                        rcu_dereference(tsk->real_parent)->mm == tsk->mm;
        rcu_read_unlock();

        return ret;
}

/*
 * Applies per-task gfp context to the given allocation flags.
 * PF_MEMALLOC_NOIO implies GFP_NOIO
 * PF_MEMALLOC_NOFS implies GFP_NOFS
 * PF_MEMALLOC_PIN  implies !GFP_MOVABLE
 */
static inline gfp_t current_gfp_context(gfp_t flags)
{
        unsigned int pflags = READ_ONCE(current->flags);

        if (unlikely(pflags & (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_PIN))) {
                /*
                 * NOIO implies both NOIO and NOFS and it is a weaker context
                 * so always make sure it makes precedence
                 */
                if (pflags & PF_MEMALLOC_NOIO)
                        flags &= ~(__GFP_IO | __GFP_FS);
                else if (pflags & PF_MEMALLOC_NOFS)
                        flags &= ~__GFP_FS;

                if (pflags & PF_MEMALLOC_PIN)
                        flags &= ~__GFP_MOVABLE;
        }
        return flags;
}

#ifdef CONFIG_LOCKDEP
extern void __fs_reclaim_acquire(unsigned long ip);
extern void __fs_reclaim_release(unsigned long ip);
extern void fs_reclaim_acquire(gfp_t gfp_mask);
extern void fs_reclaim_release(gfp_t gfp_mask);
#else
static inline void __fs_reclaim_acquire(unsigned long ip) { }
static inline void __fs_reclaim_release(unsigned long ip) { }
static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }
static inline void fs_reclaim_release(gfp_t gfp_mask) { }
#endif

/* Any memory-allocation retry loop should use
 * memalloc_retry_wait(), and pass the flags for the most
 * constrained allocation attempt that might have failed.
 * This provides useful documentation of where loops are,
 * and a central place to fine tune the waiting as the MM
 * implementation changes.
 */
static inline void memalloc_retry_wait(gfp_t gfp_flags)
{
        /* We use io_schedule_timeout because waiting for memory
         * typically included waiting for dirty pages to be
         * written out, which requires IO.
         */
        __set_current_state(TASK_UNINTERRUPTIBLE);
        gfp_flags = current_gfp_context(gfp_flags);
        if (gfpflags_allow_blocking(gfp_flags) &&
            !(gfp_flags & __GFP_NORETRY))
                /* Probably waited already, no need for much more */
                io_schedule_timeout(1);
        else
                /* Probably didn't wait, and has now released a lock,
                 * so now is a good time to wait
                 */
                io_schedule_timeout(HZ/50);
}

/**
 * might_alloc - Mark possible allocation sites
 * @gfp_mask: gfp_t flags that would be used to allocate
 *
 * Similar to might_sleep() and other annotations, this can be used in functions
 * that might allocate, but often don't. Compiles to nothing without
 * CONFIG_LOCKDEP. Includes a conditional might_sleep() if @gfp allows blocking.
 */
static inline void might_alloc(gfp_t gfp_mask)
{
        fs_reclaim_acquire(gfp_mask);
        fs_reclaim_release(gfp_mask);

        might_sleep_if(gfpflags_allow_blocking(gfp_mask));
}

/**
 * memalloc_flags_save - Add a PF_* flag to current->flags, save old value
 *
 * This allows PF_* flags to be conveniently added, irrespective of current
 * value, and then the old version restored with memalloc_flags_restore().
 */
static inline unsigned memalloc_flags_save(unsigned flags)
{
        unsigned oldflags = ~current->flags & flags;
        current->flags |= flags;
        return oldflags;
}

static inline void memalloc_flags_restore(unsigned flags)
{
        current->flags &= ~flags;
}

/**
 * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope.
 *
 * This functions marks the beginning of the GFP_NOIO allocation scope.
 * All further allocations will implicitly drop __GFP_IO flag and so
 * they are safe for the IO critical section from the allocation recursion
 * point of view. Use memalloc_noio_restore to end the scope with flags
 * returned by this function.
 *
 * Context: This function is safe to be used from any context.
 * Return: The saved flags to be passed to memalloc_noio_restore.
 */
static inline unsigned int memalloc_noio_save(void)
{
        return memalloc_flags_save(PF_MEMALLOC_NOIO);
}

/**
 * memalloc_noio_restore - Ends the implicit GFP_NOIO scope.
 * @flags: Flags to restore.
 *
 * Ends the implicit GFP_NOIO scope started by memalloc_noio_save function.
 * Always make sure that the given flags is the return value from the
 * pairing memalloc_noio_save call.
 */
static inline void memalloc_noio_restore(unsigned int flags)
{
        memalloc_flags_restore(flags);
}

/**
 * memalloc_nofs_save - Marks implicit GFP_NOFS allocation scope.
 *
 * This functions marks the beginning of the GFP_NOFS allocation scope.
 * All further allocations will implicitly drop __GFP_FS flag and so
 * they are safe for the FS critical section from the allocation recursion
 * point of view. Use memalloc_nofs_restore to end the scope with flags
 * returned by this function.
 *
 * Context: This function is safe to be used from any context.
 * Return: The saved flags to be passed to memalloc_nofs_restore.
 */
static inline unsigned int memalloc_nofs_save(void)
{
        return memalloc_flags_save(PF_MEMALLOC_NOFS);
}

/**
 * memalloc_nofs_restore - Ends the implicit GFP_NOFS scope.
 * @flags: Flags to restore.
 *
 * Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function.
 * Always make sure that the given flags is the return value from the
 * pairing memalloc_nofs_save call.
 */
static inline void memalloc_nofs_restore(unsigned int flags)
{
        memalloc_flags_restore(flags);
}

/**
 * memalloc_noreclaim_save - Marks implicit __GFP_MEMALLOC scope.
 *
 * This function marks the beginning of the __GFP_MEMALLOC allocation scope.
 * All further allocations will implicitly add the __GFP_MEMALLOC flag, which
 * prevents entering reclaim and allows access to all memory reserves. This
 * should only be used when the caller guarantees the allocation will allow more
 * memory to be freed very shortly, i.e. it needs to allocate some memory in
 * the process of freeing memory, and cannot reclaim due to potential recursion.
 *
 * Users of this scope have to be extremely careful to not deplete the reserves
 * completely and implement a throttling mechanism which controls the
 * consumption of the reserve based on the amount of freed memory. Usage of a
 * pre-allocated pool (e.g. mempool) should be always considered before using
 * this scope.
 *
 * Individual allocations under the scope can opt out using __GFP_NOMEMALLOC
 *
 * Context: This function should not be used in an interrupt context as that one
 *          does not give PF_MEMALLOC access to reserves.
 *          See __gfp_pfmemalloc_flags().
 * Return: The saved flags to be passed to memalloc_noreclaim_restore.
 */
static inline unsigned int memalloc_noreclaim_save(void)
{
        return memalloc_flags_save(PF_MEMALLOC);
}

/**
 * memalloc_noreclaim_restore - Ends the implicit __GFP_MEMALLOC scope.
 * @flags: Flags to restore.
 *
 * Ends the implicit __GFP_MEMALLOC scope started by memalloc_noreclaim_save
 * function. Always make sure that the given flags is the return value from the
 * pairing memalloc_noreclaim_save call.
 */
static inline void memalloc_noreclaim_restore(unsigned int flags)
{
        memalloc_flags_restore(flags);
}

/**
 * memalloc_pin_save - Marks implicit ~__GFP_MOVABLE scope.
 *
 * This function marks the beginning of the ~__GFP_MOVABLE allocation scope.
 * All further allocations will implicitly remove the __GFP_MOVABLE flag, which
 * will constraint the allocations to zones that allow long term pinning, i.e.
 * not ZONE_MOVABLE zones.
 *
 * Return: The saved flags to be passed to memalloc_pin_restore.
 */
static inline unsigned int memalloc_pin_save(void)
{
        return memalloc_flags_save(PF_MEMALLOC_PIN);
}

/**
 * memalloc_pin_restore - Ends the implicit ~__GFP_MOVABLE scope.
 * @flags: Flags to restore.
 *
 * Ends the implicit ~__GFP_MOVABLE scope started by memalloc_pin_save function.
 * Always make sure that the given flags is the return value from the pairing
 * memalloc_pin_save call.
 */
static inline void memalloc_pin_restore(unsigned int flags)
{
        memalloc_flags_restore(flags);
}

#ifdef CONFIG_MEMCG
DECLARE_PER_CPU(struct mem_cgroup *, int_active_memcg);
/**
 * set_active_memcg - Starts the remote memcg charging scope.
 * @memcg: memcg to charge.
 *
 * This function marks the beginning of the remote memcg charging scope. All the
 * __GFP_ACCOUNT allocations till the end of the scope will be charged to the
 * given memcg.
 *
 * Please, make sure that caller has a reference to the passed memcg structure,
 * so its lifetime is guaranteed to exceed the scope between two
 * set_active_memcg() calls.
 *
 * NOTE: This function can nest. Users must save the return value and
 * reset the previous value after their own charging scope is over.
 */
static inline struct mem_cgroup *
set_active_memcg(struct mem_cgroup *memcg)
{
        struct mem_cgroup *old;

        if (!in_task()) {
                old = this_cpu_read(int_active_memcg);
                this_cpu_write(int_active_memcg, memcg);
        } else {
                old = current->active_memcg;
                current->active_memcg = memcg;
        }

        return old;
}
#else
static inline struct mem_cgroup *
set_active_memcg(struct mem_cgroup *memcg)
{
        return NULL;
}
#endif

#ifdef CONFIG_MEMBARRIER
enum {
        MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY                = (1U << 0),
        MEMBARRIER_STATE_PRIVATE_EXPEDITED                        = (1U << 1),
        MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY                        = (1U << 2),
        MEMBARRIER_STATE_GLOBAL_EXPEDITED                        = (1U << 3),
        MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY        = (1U << 4),
        MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE                = (1U << 5),
        MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY                = (1U << 6),
        MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ                        = (1U << 7),
};

enum {
        MEMBARRIER_FLAG_SYNC_CORE        = (1U << 0),
        MEMBARRIER_FLAG_RSEQ                = (1U << 1),
};

#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
#include <asm/membarrier.h>
#endif

static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
{
        /*
         * The atomic_read() below prevents CSE. The following should
         * help the compiler generate more efficient code on architectures
         * where sync_core_before_usermode() is a no-op.
         */
        if (!IS_ENABLED(CONFIG_ARCH_HAS_SYNC_CORE_BEFORE_USERMODE))
                return;
        if (current->mm != mm)
                return;
        if (likely(!(atomic_read(&mm->membarrier_state) &
                     MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE)))
                return;
        sync_core_before_usermode();
}

extern void membarrier_exec_mmap(struct mm_struct *mm);

extern void membarrier_update_current_mm(struct mm_struct *next_mm);

#else
#ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
                                             struct mm_struct *next,
                                             struct task_struct *tsk)
{
}
#endif
static inline void membarrier_exec_mmap(struct mm_struct *mm)
{
}
static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
{
}
static inline void membarrier_update_current_mm(struct mm_struct *next_mm)
{
}
#endif

#endif /* _LINUX_SCHED_MM_H */























  517 









































































































    2 






    2 
   71 



    2 
   71 




    1 












    1 

























   88 
















































    2 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the UDP protocol.
 *
 * Version:        @(#)udp.h        1.0.2        04/28/93
 *
 * Author:        Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 */
#ifndef _LINUX_UDP_H
#define _LINUX_UDP_H

#include <net/inet_sock.h>
#include <linux/skbuff.h>
#include <net/netns/hash.h>
#include <uapi/linux/udp.h>

static inline struct udphdr *udp_hdr(const struct sk_buff *skb)
{
        return (struct udphdr *)skb_transport_header(skb);
}

#define UDP_HTABLE_SIZE_MIN_PERNET        128
#define UDP_HTABLE_SIZE_MIN                (IS_ENABLED(CONFIG_BASE_SMALL) ? 128 : 256)
#define UDP_HTABLE_SIZE_MAX                65536

static inline u32 udp_hashfn(const struct net *net, u32 num, u32 mask)
{
        return (num + net_hash_mix(net)) & mask;
}

enum {
        UDP_FLAGS_CORK,                /* Cork is required */
        UDP_FLAGS_NO_CHECK6_TX, /* Send zero UDP6 checksums on TX? */
        UDP_FLAGS_NO_CHECK6_RX, /* Allow zero UDP6 checksums on RX? */
        UDP_FLAGS_GRO_ENABLED,        /* Request GRO aggregation */
        UDP_FLAGS_ACCEPT_FRAGLIST,
        UDP_FLAGS_ACCEPT_L4,
        UDP_FLAGS_ENCAP_ENABLED, /* This socket enabled encap */
        UDP_FLAGS_UDPLITE_SEND_CC, /* set via udplite setsockopt */
        UDP_FLAGS_UDPLITE_RECV_CC, /* set via udplite setsockopt */
};

struct udp_sock {
        /* inet_sock has to be the first member */
        struct inet_sock inet;
#define udp_port_hash                inet.sk.__sk_common.skc_u16hashes[0]
#define udp_portaddr_hash        inet.sk.__sk_common.skc_u16hashes[1]
#define udp_portaddr_node        inet.sk.__sk_common.skc_portaddr_node

        unsigned long         udp_flags;

        int                 pending;        /* Any pending frames ? */
        __u8                 encap_type;        /* Is this an Encapsulation socket? */

#if !IS_ENABLED(CONFIG_BASE_SMALL)
        /* For UDP 4-tuple hash */
        __u16 udp_lrpa_hash;
        struct hlist_nulls_node udp_lrpa_node;
#endif

        /*
         * Following member retains the information to create a UDP header
         * when the socket is uncorked.
         */
        __u16                 len;                /* total length of pending frames */
        __u16                 gso_size;
        /*
         * Fields specific to UDP-Lite.
         */
        __u16                 pcslen;
        __u16                 pcrlen;
        /*
         * For encapsulation sockets.
         */
        int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
        void (*encap_err_rcv)(struct sock *sk, struct sk_buff *skb, int err,
                              __be16 port, u32 info, u8 *payload);
        int (*encap_err_lookup)(struct sock *sk, struct sk_buff *skb);
        void (*encap_destroy)(struct sock *sk);

        /* GRO functions for UDP socket */
        struct sk_buff *        (*gro_receive)(struct sock *sk,
                                               struct list_head *head,
                                               struct sk_buff *skb);
        int                        (*gro_complete)(struct sock *sk,
                                                struct sk_buff *skb,
                                                int nhoff);

        /* udp_recvmsg try to use this before splicing sk_receive_queue */
        struct sk_buff_head        reader_queue ____cacheline_aligned_in_smp;

        /* This field is dirtied by udp_recvmsg() */
        int                forward_deficit;

        /* This fields follows rcvbuf value, and is touched by udp_recvmsg */
        int                forward_threshold;

        /* Cache friendly copy of sk->sk_peek_off >= 0 */
        bool                peeking_with_offset;

        /*
         * Accounting for the tunnel GRO fastpath.
         * Unprotected by compilers guard, as it uses space available in
         * the last UDP socket cacheline.
         */
        struct hlist_node        tunnel_list;
};

#define udp_test_bit(nr, sk)                        \
        test_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
#define udp_set_bit(nr, sk)                        \
        set_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
#define udp_test_and_set_bit(nr, sk)                \
        test_and_set_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
#define udp_clear_bit(nr, sk)                        \
        clear_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags)
#define udp_assign_bit(nr, sk, val)                \
        assign_bit(UDP_FLAGS_##nr, &udp_sk(sk)->udp_flags, val)

#define UDP_MAX_SEGMENTS        (1 << 7UL)

#define udp_sk(ptr) container_of_const(ptr, struct udp_sock, inet.sk)

static inline int udp_set_peek_off(struct sock *sk, int val)
{
        sk_set_peek_off(sk, val);
        WRITE_ONCE(udp_sk(sk)->peeking_with_offset, val >= 0);
        return 0;
}

static inline void udp_set_no_check6_tx(struct sock *sk, bool val)
{
        udp_assign_bit(NO_CHECK6_TX, sk, val);
}

static inline void udp_set_no_check6_rx(struct sock *sk, bool val)
{
        udp_assign_bit(NO_CHECK6_RX, sk, val);
}

static inline bool udp_get_no_check6_tx(const struct sock *sk)
{
        return udp_test_bit(NO_CHECK6_TX, sk);
}

static inline bool udp_get_no_check6_rx(const struct sock *sk)
{
        return udp_test_bit(NO_CHECK6_RX, sk);
}

static inline void udp_cmsg_recv(struct msghdr *msg, struct sock *sk,
                                 struct sk_buff *skb)
{
        int gso_size;

        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
                gso_size = skb_shinfo(skb)->gso_size;
                put_cmsg(msg, SOL_UDP, UDP_GRO, sizeof(gso_size), &gso_size);
        }
}

DECLARE_STATIC_KEY_FALSE(udp_encap_needed_key);
#if IS_ENABLED(CONFIG_IPV6)
DECLARE_STATIC_KEY_FALSE(udpv6_encap_needed_key);
#endif

static inline bool udp_encap_needed(void)
{
        if (static_branch_unlikely(&udp_encap_needed_key))
                return true;

#if IS_ENABLED(CONFIG_IPV6)
        if (static_branch_unlikely(&udpv6_encap_needed_key))
                return true;
#endif

        return false;
}

static inline bool udp_unexpected_gso(struct sock *sk, struct sk_buff *skb)
{
        if (!skb_is_gso(skb))
                return false;

        if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 &&
            !udp_test_bit(ACCEPT_L4, sk))
                return true;

        if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST &&
            !udp_test_bit(ACCEPT_FRAGLIST, sk))
                return true;

        /* GSO packets lacking the SKB_GSO_UDP_TUNNEL/_CSUM bits might still
         * land in a tunnel as the socket check in udp_gro_receive cannot be
         * foolproof.
         */
        if (udp_encap_needed() &&
            READ_ONCE(udp_sk(sk)->encap_rcv) &&
            !(skb_shinfo(skb)->gso_type &
              (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)))
                return true;

        return false;
}

static inline void udp_allow_gso(struct sock *sk)
{
        udp_set_bit(ACCEPT_L4, sk);
        udp_set_bit(ACCEPT_FRAGLIST, sk);
}

#define udp_portaddr_for_each_entry(__sk, list) \
        hlist_for_each_entry(__sk, list, __sk_common.skc_portaddr_node)

#define udp_portaddr_for_each_entry_from(__sk) \
        hlist_for_each_entry_from(__sk, __sk_common.skc_portaddr_node)

#define udp_portaddr_for_each_entry_rcu(__sk, list) \
        hlist_for_each_entry_rcu(__sk, list, __sk_common.skc_portaddr_node)

#if !IS_ENABLED(CONFIG_BASE_SMALL)
#define udp_lrpa_for_each_entry_rcu(__up, node, list) \
        hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node)
#endif

#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)

static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
{
#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
        return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);
#else
        return NULL;
#endif
}

#endif        /* _LINUX_UDP_H */












































































































































































































































































































































































































































































































   26 




























































   16 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* Network filesystem support services.
 *
 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * See:
 *
 *        Documentation/filesystems/netfs_library.rst
 *
 * for a description of the network filesystem interface declared here.
 */

#ifndef _LINUX_NETFS_H
#define _LINUX_NETFS_H

#include <linux/workqueue.h>
#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/uio.h>
#include <linux/rolling_buffer.h>

enum netfs_sreq_ref_trace;
typedef struct mempool_s mempool_t;
struct folio_queue;

/**
 * folio_start_private_2 - Start an fscache write on a folio.  [DEPRECATED]
 * @folio: The folio.
 *
 * Call this function before writing a folio to a local cache.  Starting a
 * second write before the first one finishes is not allowed.
 *
 * Note that this should no longer be used.
 */
static inline void folio_start_private_2(struct folio *folio)
{
        VM_BUG_ON_FOLIO(folio_test_private_2(folio), folio);
        folio_get(folio);
        folio_set_private_2(folio);
}

enum netfs_io_source {
        NETFS_SOURCE_UNKNOWN,
        NETFS_FILL_WITH_ZEROES,
        NETFS_DOWNLOAD_FROM_SERVER,
        NETFS_READ_FROM_CACHE,
        NETFS_INVALID_READ,
        NETFS_UPLOAD_TO_SERVER,
        NETFS_WRITE_TO_CACHE,
} __mode(byte);

typedef void (*netfs_io_terminated_t)(void *priv, ssize_t transferred_or_error);

/*
 * Per-inode context.  This wraps the VFS inode.
 */
struct netfs_inode {
        struct inode                inode;                /* The VFS inode */
        const struct netfs_request_ops *ops;
#if IS_ENABLED(CONFIG_FSCACHE)
        struct fscache_cookie        *cache;
#endif
        struct mutex                wb_lock;        /* Writeback serialisation */
        loff_t                        remote_i_size;        /* Size of the remote file */
        loff_t                        zero_point;        /* Size after which we assume there's no data
                                                 * on the server */
        atomic_t                io_count;        /* Number of outstanding reqs */
        unsigned long                flags;
#define NETFS_ICTX_ODIRECT        0                /* The file has DIO in progress */
#define NETFS_ICTX_UNBUFFERED        1                /* I/O should not use the pagecache */
#define NETFS_ICTX_MODIFIED_ATTR 3                /* Indicate change in mtime/ctime */
#define NETFS_ICTX_SINGLE_NO_UPLOAD 4                /* Monolithic payload, cache but no upload */
};

/*
 * A netfs group - for instance a ceph snap.  This is marked on dirty pages and
 * pages marked with a group must be flushed before they can be written under
 * the domain of another group.
 */
struct netfs_group {
        refcount_t                ref;
        void (*free)(struct netfs_group *netfs_group);
};

/*
 * Information about a dirty page (attached only if necessary).
 * folio->private
 */
struct netfs_folio {
        struct netfs_group        *netfs_group;        /* Filesystem's grouping marker (or NULL). */
        unsigned int                dirty_offset;        /* Write-streaming dirty data offset */
        unsigned int                dirty_len;        /* Write-streaming dirty data length */
};
#define NETFS_FOLIO_INFO        0x1UL        /* OR'd with folio->private. */
#define NETFS_FOLIO_COPY_TO_CACHE ((struct netfs_group *)0x356UL) /* Write to the cache only */

static inline bool netfs_is_folio_info(const void *priv)
{
        return (unsigned long)priv & NETFS_FOLIO_INFO;
}

static inline struct netfs_folio *__netfs_folio_info(const void *priv)
{
        if (netfs_is_folio_info(priv))
                return (struct netfs_folio *)((unsigned long)priv & ~NETFS_FOLIO_INFO);
        return NULL;
}

static inline struct netfs_folio *netfs_folio_info(struct folio *folio)
{
        return __netfs_folio_info(folio_get_private(folio));
}

static inline struct netfs_group *netfs_folio_group(struct folio *folio)
{
        struct netfs_folio *finfo;
        void *priv = folio_get_private(folio);

        finfo = netfs_folio_info(folio);
        if (finfo)
                return finfo->netfs_group;
        return priv;
}

/*
 * Stream of I/O subrequests going to a particular destination, such as the
 * server or the local cache.  This is mainly intended for writing where we may
 * have to write to multiple destinations concurrently.
 */
struct netfs_io_stream {
        /* Submission tracking */
        struct netfs_io_subrequest *construct;        /* Op being constructed */
        size_t                        sreq_max_len;        /* Maximum size of a subrequest */
        unsigned int                sreq_max_segs;        /* 0 or max number of segments in an iterator */
        unsigned int                submit_off;        /* Folio offset we're submitting from */
        unsigned int                submit_len;        /* Amount of data left to submit */
        unsigned int                submit_extendable_to; /* Amount I/O can be rounded up to */
        void (*prepare_write)(struct netfs_io_subrequest *subreq);
        void (*issue_write)(struct netfs_io_subrequest *subreq);
        /* Collection tracking */
        struct list_head        subrequests;        /* Contributory I/O operations */
        struct netfs_io_subrequest *front;        /* Op being collected */
        unsigned long long        collected_to;        /* Position we've collected results to */
        size_t                        transferred;        /* The amount transferred from this stream */
        unsigned short                error;                /* Aggregate error for the stream */
        enum netfs_io_source        source;                /* Where to read from/write to */
        unsigned char                stream_nr;        /* Index of stream in parent table */
        bool                        avail;                /* T if stream is available */
        bool                        active;                /* T if stream is active */
        bool                        need_retry;        /* T if this stream needs retrying */
        bool                        failed;                /* T if this stream failed */
};

/*
 * Resources required to do operations on a cache.
 */
struct netfs_cache_resources {
        const struct netfs_cache_ops        *ops;
        void                                *cache_priv;
        void                                *cache_priv2;
        unsigned int                        debug_id;        /* Cookie debug ID */
        unsigned int                        inval_counter;        /* object->inval_counter at begin_op */
};

/*
 * Descriptor for a single component subrequest.  Each operation represents an
 * individual read/write from/to a server, a cache, a journal, etc..
 *
 * The buffer iterator is persistent for the life of the subrequest struct and
 * the pages it points to can be relied on to exist for the duration.
 */
struct netfs_io_subrequest {
        struct netfs_io_request *rreq;                /* Supervising I/O request */
        struct work_struct        work;
        struct list_head        rreq_link;        /* Link in rreq->subrequests */
        struct iov_iter                io_iter;        /* Iterator for this subrequest */
        unsigned long long        start;                /* Where to start the I/O */
        size_t                        len;                /* Size of the I/O */
        size_t                        transferred;        /* Amount of data transferred */
        refcount_t                ref;
        short                        error;                /* 0 or error that occurred */
        unsigned short                debug_index;        /* Index in list (for debugging output) */
        unsigned int                nr_segs;        /* Number of segs in io_iter */
        u8                        retry_count;        /* The number of retries (0 on initial pass) */
        enum netfs_io_source        source;                /* Where to read from/write to */
        unsigned char                stream_nr;        /* I/O stream this belongs to */
        unsigned long                flags;
#define NETFS_SREQ_COPY_TO_CACHE        0        /* Set if should copy the data to the cache */
#define NETFS_SREQ_CLEAR_TAIL                1        /* Set if the rest of the read should be cleared */
#define NETFS_SREQ_MADE_PROGRESS        4        /* Set if we transferred at least some data */
#define NETFS_SREQ_ONDEMAND                5        /* Set if it's from on-demand read mode */
#define NETFS_SREQ_BOUNDARY                6        /* Set if ends on hard boundary (eg. ceph object) */
#define NETFS_SREQ_HIT_EOF                7        /* Set if short due to EOF */
#define NETFS_SREQ_IN_PROGRESS                8        /* Unlocked when the subrequest completes */
#define NETFS_SREQ_NEED_RETRY                9        /* Set if the filesystem requests a retry */
#define NETFS_SREQ_FAILED                10        /* Set if the subreq failed unretryably */
};

enum netfs_io_origin {
        NETFS_READAHEAD,                /* This read was triggered by readahead */
        NETFS_READPAGE,                        /* This read is a synchronous read */
        NETFS_READ_GAPS,                /* This read is a synchronous read to fill gaps */
        NETFS_READ_SINGLE,                /* This read should be treated as a single object */
        NETFS_READ_FOR_WRITE,                /* This read is to prepare a write */
        NETFS_UNBUFFERED_READ,                /* This is an unbuffered read */
        NETFS_DIO_READ,                        /* This is a direct I/O read */
        NETFS_WRITEBACK,                /* This write was triggered by writepages */
        NETFS_WRITEBACK_SINGLE,                /* This monolithic write was triggered by writepages */
        NETFS_WRITETHROUGH,                /* This write was made by netfs_perform_write() */
        NETFS_UNBUFFERED_WRITE,                /* This is an unbuffered write */
        NETFS_DIO_WRITE,                /* This is a direct I/O write */
        NETFS_PGPRIV2_COPY_TO_CACHE,        /* [DEPRECATED] This is writing read data to the cache */
        nr__netfs_io_origin
} __mode(byte);

/*
 * Descriptor for an I/O helper request.  This is used to make multiple I/O
 * operations to a variety of data stores and then stitch the result together.
 */
struct netfs_io_request {
        union {
                struct work_struct cleanup_work; /* Deferred cleanup work */
                struct rcu_head rcu;
        };
        struct work_struct        work;                /* Result collector work */
        struct inode                *inode;                /* The file being accessed */
        struct address_space        *mapping;        /* The mapping being accessed */
        struct kiocb                *iocb;                /* AIO completion vector */
        struct netfs_cache_resources cache_resources;
        struct netfs_io_request        *copy_to_cache;        /* Request to write just-read data to the cache */
#ifdef CONFIG_PROC_FS
        struct list_head        proc_link;        /* Link in netfs_iorequests */
#endif
        struct netfs_io_stream        io_streams[2];        /* Streams of parallel I/O operations */
#define NR_IO_STREAMS 2 //wreq->nr_io_streams
        struct netfs_group        *group;                /* Writeback group being written back */
        struct rolling_buffer        buffer;                /* Unencrypted buffer */
#define NETFS_ROLLBUF_PUT_MARK                ROLLBUF_MARK_1
#define NETFS_ROLLBUF_PAGECACHE_MARK        ROLLBUF_MARK_2
        wait_queue_head_t        waitq;                /* Processor waiter */
        void                        *netfs_priv;        /* Private data for the netfs */
        void                        *netfs_priv2;        /* Private data for the netfs */
        struct bio_vec                *direct_bv;        /* DIO buffer list (when handling iovec-iter) */
        unsigned long long        submitted;        /* Amount submitted for I/O so far */
        unsigned long long        len;                /* Length of the request */
        size_t                        transferred;        /* Amount to be indicated as transferred */
        long                        error;                /* 0 or error that occurred */
        unsigned long long        i_size;                /* Size of the file */
        unsigned long long        start;                /* Start position */
        atomic64_t                issued_to;        /* Write issuer folio cursor */
        unsigned long long        collected_to;        /* Point we've collected to */
        unsigned long long        cleaned_to;        /* Position we've cleaned folios to */
        unsigned long long        abandon_to;        /* Position to abandon folios to */
        pgoff_t                        no_unlock_folio; /* Don't unlock this folio after read */
        unsigned int                direct_bv_count; /* Number of elements in direct_bv[] */
        unsigned int                debug_id;
        unsigned int                rsize;                /* Maximum read size (0 for none) */
        unsigned int                wsize;                /* Maximum write size (0 for none) */
        atomic_t                subreq_counter;        /* Next subreq->debug_index */
        unsigned int                nr_group_rel;        /* Number of refs to release on ->group */
        spinlock_t                lock;                /* Lock for queuing subreqs */
        unsigned char                front_folio_order; /* Order (size) of front folio */
        enum netfs_io_origin        origin;                /* Origin of the request */
        bool                        direct_bv_unpin; /* T if direct_bv[] must be unpinned */
        refcount_t                ref;
        unsigned long                flags;
#define NETFS_RREQ_OFFLOAD_COLLECTION        0        /* Offload collection to workqueue */
#define NETFS_RREQ_NO_UNLOCK_FOLIO        2        /* Don't unlock no_unlock_folio on completion */
#define NETFS_RREQ_FAILED                4        /* The request failed */
#define NETFS_RREQ_IN_PROGRESS                5        /* Unlocked when the request completes (has ref) */
#define NETFS_RREQ_FOLIO_COPY_TO_CACHE        6        /* Copy current folio to cache from read */
#define NETFS_RREQ_UPLOAD_TO_SERVER        8        /* Need to write to the server */
#define NETFS_RREQ_PAUSE                11        /* Pause subrequest generation */
#define NETFS_RREQ_USE_IO_ITER                12        /* Use ->io_iter rather than ->i_pages */
#define NETFS_RREQ_ALL_QUEUED                13        /* All subreqs are now queued */
#define NETFS_RREQ_RETRYING                14        /* Set if we're in the retry path */
#define NETFS_RREQ_SHORT_TRANSFER        15        /* Set if we have a short transfer */
#define NETFS_RREQ_USE_PGPRIV2                31        /* [DEPRECATED] Use PG_private_2 to mark
                                                 * write to cache on read */
        const struct netfs_request_ops *netfs_ops;
        void (*cleanup)(struct netfs_io_request *req);
};

/*
 * Operations the network filesystem can/must provide to the helpers.
 */
struct netfs_request_ops {
        mempool_t *request_pool;
        mempool_t *subrequest_pool;
        int (*init_request)(struct netfs_io_request *rreq, struct file *file);
        void (*free_request)(struct netfs_io_request *rreq);
        void (*free_subrequest)(struct netfs_io_subrequest *rreq);

        /* Read request handling */
        void (*expand_readahead)(struct netfs_io_request *rreq);
        int (*prepare_read)(struct netfs_io_subrequest *subreq);
        void (*issue_read)(struct netfs_io_subrequest *subreq);
        bool (*is_still_valid)(struct netfs_io_request *rreq);
        int (*check_write_begin)(struct file *file, loff_t pos, unsigned len,
                                 struct folio **foliop, void **_fsdata);
        void (*done)(struct netfs_io_request *rreq);

        /* Modification handling */
        void (*update_i_size)(struct inode *inode, loff_t i_size);
        void (*post_modify)(struct inode *inode);

        /* Write request handling */
        void (*begin_writeback)(struct netfs_io_request *wreq);
        void (*prepare_write)(struct netfs_io_subrequest *subreq);
        void (*issue_write)(struct netfs_io_subrequest *subreq);
        void (*retry_request)(struct netfs_io_request *wreq, struct netfs_io_stream *stream);
        void (*invalidate_cache)(struct netfs_io_request *wreq);
};

/*
 * How to handle reading from a hole.
 */
enum netfs_read_from_hole {
        NETFS_READ_HOLE_IGNORE,
        NETFS_READ_HOLE_FAIL,
};

/*
 * Table of operations for access to a cache.
 */
struct netfs_cache_ops {
        /* End an operation */
        void (*end_operation)(struct netfs_cache_resources *cres);

        /* Read data from the cache */
        int (*read)(struct netfs_cache_resources *cres,
                    loff_t start_pos,
                    struct iov_iter *iter,
                    enum netfs_read_from_hole read_hole,
                    netfs_io_terminated_t term_func,
                    void *term_func_priv);

        /* Write data to the cache */
        int (*write)(struct netfs_cache_resources *cres,
                     loff_t start_pos,
                     struct iov_iter *iter,
                     netfs_io_terminated_t term_func,
                     void *term_func_priv);

        /* Write data to the cache from a netfs subrequest. */
        void (*issue_write)(struct netfs_io_subrequest *subreq);

        /* Expand readahead request */
        void (*expand_readahead)(struct netfs_cache_resources *cres,
                                 unsigned long long *_start,
                                 unsigned long long *_len,
                                 unsigned long long i_size);

        /* Prepare a read operation, shortening it to a cached/uncached
         * boundary as appropriate.
         */
        enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq,
                                             unsigned long long i_size);

        /* Prepare a write subrequest, working out if we're allowed to do it
         * and finding out the maximum amount of data to gather before
         * attempting to submit.  If we're not permitted to do it, the
         * subrequest should be marked failed.
         */
        void (*prepare_write_subreq)(struct netfs_io_subrequest *subreq);

        /* Prepare a write operation, working out what part of the write we can
         * actually do.
         */
        int (*prepare_write)(struct netfs_cache_resources *cres,
                             loff_t *_start, size_t *_len, size_t upper_len,
                             loff_t i_size, bool no_space_allocated_yet);

        /* Prepare an on-demand read operation, shortening it to a cached/uncached
         * boundary as appropriate.
         */
        enum netfs_io_source (*prepare_ondemand_read)(struct netfs_cache_resources *cres,
                                                      loff_t start, size_t *_len,
                                                      loff_t i_size,
                                                      unsigned long *_flags, ino_t ino);

        /* Query the occupancy of the cache in a region, returning where the
         * next chunk of data starts and how long it is.
         */
        int (*query_occupancy)(struct netfs_cache_resources *cres,
                               loff_t start, size_t len, size_t granularity,
                               loff_t *_data_start, size_t *_data_len);
};

/* High-level read API. */
ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter);
ssize_t netfs_unbuffered_read_iter(struct kiocb *iocb, struct iov_iter *iter);
ssize_t netfs_buffered_read_iter(struct kiocb *iocb, struct iov_iter *iter);
ssize_t netfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter);

/* High-level write API */
ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
                            struct netfs_group *netfs_group);
ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from,
                                         struct netfs_group *netfs_group);
ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from);
ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *iter,
                                           struct netfs_group *netfs_group);
ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from);

/* Single, monolithic object read/write API. */
void netfs_single_mark_inode_dirty(struct inode *inode);
ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_iter *iter);
int netfs_writeback_single(struct address_space *mapping,
                           struct writeback_control *wbc,
                           struct iov_iter *iter);

/* Address operations API */
struct readahead_control;
void netfs_readahead(struct readahead_control *);
int netfs_read_folio(struct file *, struct folio *);
int netfs_write_begin(struct netfs_inode *, struct file *,
                      struct address_space *, loff_t pos, unsigned int len,
                      struct folio **, void **fsdata);
int netfs_writepages(struct address_space *mapping,
                     struct writeback_control *wbc);
bool netfs_dirty_folio(struct address_space *mapping, struct folio *folio);
int netfs_unpin_writeback(struct inode *inode, struct writeback_control *wbc);
void netfs_clear_inode_writeback(struct inode *inode, const void *aux);
void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length);
bool netfs_release_folio(struct folio *folio, gfp_t gfp);

/* VMA operations API. */
vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group);

/* (Sub)request management API. */
void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq);
void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq);
void netfs_get_subrequest(struct netfs_io_subrequest *subreq,
                          enum netfs_sreq_ref_trace what);
void netfs_put_subrequest(struct netfs_io_subrequest *subreq,
                          enum netfs_sreq_ref_trace what);
ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
                                struct iov_iter *new,
                                iov_iter_extraction_t extraction_flags);
size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset,
                        size_t max_size, size_t max_segs);
void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq);
void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error);
void netfs_queue_write_request(struct netfs_io_subrequest *subreq);

int netfs_start_io_read(struct inode *inode);
void netfs_end_io_read(struct inode *inode);
int netfs_start_io_write(struct inode *inode);
void netfs_end_io_write(struct inode *inode);
int netfs_start_io_direct(struct inode *inode);
void netfs_end_io_direct(struct inode *inode);

/* Miscellaneous APIs. */
struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp,
                                       unsigned int trace /*enum netfs_folioq_trace*/);
void netfs_folioq_free(struct folio_queue *folioq,
                       unsigned int trace /*enum netfs_trace_folioq*/);

/* Buffer wrangling helpers API. */
int netfs_alloc_folioq_buffer(struct address_space *mapping,
                              struct folio_queue **_buffer,
                              size_t *_cur_size, ssize_t size, gfp_t gfp);
void netfs_free_folioq_buffer(struct folio_queue *fq);

/**
 * netfs_inode - Get the netfs inode context from the inode
 * @inode: The inode to query
 *
 * Get the netfs lib inode context from the network filesystem's inode.  The
 * context struct is expected to directly follow on from the VFS inode struct.
 */
static inline struct netfs_inode *netfs_inode(struct inode *inode)
{
        return container_of(inode, struct netfs_inode, inode);
}

/**
 * netfs_inode_init - Initialise a netfslib inode context
 * @ctx: The netfs inode to initialise
 * @ops: The netfs's operations list
 * @use_zero_point: True to use the zero_point read optimisation
 *
 * Initialise the netfs library context struct.  This is expected to follow on
 * directly from the VFS inode struct.
 */
static inline void netfs_inode_init(struct netfs_inode *ctx,
                                    const struct netfs_request_ops *ops,
                                    bool use_zero_point)
{
        ctx->ops = ops;
        ctx->remote_i_size = i_size_read(&ctx->inode);
        ctx->zero_point = LLONG_MAX;
        ctx->flags = 0;
        atomic_set(&ctx->io_count, 0);
#if IS_ENABLED(CONFIG_FSCACHE)
        ctx->cache = NULL;
#endif
        mutex_init(&ctx->wb_lock);
        /* ->releasepage() drives zero_point */
        if (use_zero_point) {
                ctx->zero_point = ctx->remote_i_size;
                mapping_set_release_always(ctx->inode.i_mapping);
        }
}

/**
 * netfs_resize_file - Note that a file got resized
 * @ctx: The netfs inode being resized
 * @new_i_size: The new file size
 * @changed_on_server: The change was applied to the server
 *
 * Inform the netfs lib that a file got resized so that it can adjust its state.
 */
static inline void netfs_resize_file(struct netfs_inode *ctx, loff_t new_i_size,
                                     bool changed_on_server)
{
        if (changed_on_server)
                ctx->remote_i_size = new_i_size;
        if (new_i_size < ctx->zero_point)
                ctx->zero_point = new_i_size;
}

/**
 * netfs_i_cookie - Get the cache cookie from the inode
 * @ctx: The netfs inode to query
 *
 * Get the caching cookie (if enabled) from the network filesystem's inode.
 */
static inline struct fscache_cookie *netfs_i_cookie(struct netfs_inode *ctx)
{
#if IS_ENABLED(CONFIG_FSCACHE)
        return ctx->cache;
#else
        return NULL;
#endif
}

/**
 * netfs_wait_for_outstanding_io - Wait for outstanding I/O to complete
 * @inode: The netfs inode to wait on
 *
 * Wait for outstanding I/O requests of any type to complete.  This is intended
 * to be called from inode eviction routines.  This makes sure that any
 * resources held by those requests are cleaned up before we let the inode get
 * cleaned up.
 */
static inline void netfs_wait_for_outstanding_io(struct inode *inode)
{
        struct netfs_inode *ictx = netfs_inode(inode);

        wait_var_event(&ictx->io_count, atomic_read(&ictx->io_count) == 0);
}

#endif /* _LINUX_NETFS_H */












































































    4 











































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FS_H
#define _BCACHEFS_FS_H

#include "inode.h"
#include "opts.h"
#include "str_hash.h"
#include "quota_types.h"
#include "two_state_shared_lock.h"

#include <linux/seqlock.h>
#include <linux/stat.h>

struct bch_inode_info {
        struct inode                v;
        struct rhash_head        hash;
        struct rhlist_head        by_inum_hash;
        subvol_inum                ei_inum;

        struct list_head        ei_vfs_inode_list;
        unsigned long                ei_flags;

        struct mutex                ei_update_lock;
        u64                        ei_quota_reserved;
        unsigned long                ei_last_dirtied;
        two_state_lock_t        ei_pagecache_lock;

        struct mutex                ei_quota_lock;
        struct bch_qid                ei_qid;

        /*
         * When we've been doing nocow writes we'll need to issue flushes to the
         * underlying block devices
         *
         * XXX: a device may have had a flush issued by some other codepath. It
         * would be better to keep for each device a sequence number that's
         * incremented when we isusue a cache flush, and track here the sequence
         * number that needs flushing.
         */
        struct bch_devs_mask        ei_devs_need_flush;

        /* copy of inode in btree: */
        struct bch_inode_unpacked ei_inode;
};

#define bch2_pagecache_add_put(i)        bch2_two_state_unlock(&i->ei_pagecache_lock, 0)
#define bch2_pagecache_add_tryget(i)        bch2_two_state_trylock(&i->ei_pagecache_lock, 0)
#define bch2_pagecache_add_get(i)        bch2_two_state_lock(&i->ei_pagecache_lock, 0)

#define bch2_pagecache_block_put(i)        bch2_two_state_unlock(&i->ei_pagecache_lock, 1)
#define bch2_pagecache_block_get(i)        bch2_two_state_lock(&i->ei_pagecache_lock, 1)

static inline subvol_inum inode_inum(struct bch_inode_info *inode)
{
        return inode->ei_inum;
}

/*
 * Set if we've gotten a btree error for this inode, and thus the vfs inode and
 * btree inode may be inconsistent:
 */
#define EI_INODE_ERROR                        0

/*
 * Set in the inode is in a snapshot subvolume - we don't do quota accounting in
 * those:
 */
#define EI_INODE_SNAPSHOT                1
#define EI_INODE_HASHED                        2

#define to_bch_ei(_inode)                                        \
        container_of_or_null(_inode, struct bch_inode_info, v)

static inline int ptrcmp(void *l, void *r)
{
        return cmp_int(l, r);
}

enum bch_inode_lock_op {
        INODE_PAGECACHE_BLOCK        = (1U << 0),
        INODE_UPDATE_LOCK        = (1U << 1),
};

#define bch2_lock_inodes(_locks, ...)                                        \
do {                                                                        \
        struct bch_inode_info *a[] = { NULL, __VA_ARGS__ };                \
        unsigned i;                                                        \
                                                                        \
        bubble_sort(&a[1], ARRAY_SIZE(a) - 1, ptrcmp);                        \
                                                                        \
        for (i = 1; i < ARRAY_SIZE(a); i++)                                \
                if (a[i] != a[i - 1]) {                                        \
                        if ((_locks) & INODE_PAGECACHE_BLOCK)                \
                                bch2_pagecache_block_get(a[i]);\
                        if ((_locks) & INODE_UPDATE_LOCK)                        \
                                mutex_lock_nested(&a[i]->ei_update_lock, i);\
                }                                                        \
} while (0)

#define bch2_unlock_inodes(_locks, ...)                                        \
do {                                                                        \
        struct bch_inode_info *a[] = { NULL, __VA_ARGS__ };                \
        unsigned i;                                                        \
                                                                        \
        bubble_sort(&a[1], ARRAY_SIZE(a) - 1, ptrcmp);                        \
                                                                        \
        for (i = 1; i < ARRAY_SIZE(a); i++)                                \
                if (a[i] != a[i - 1]) {                                        \
                        if ((_locks) & INODE_PAGECACHE_BLOCK)                \
                                bch2_pagecache_block_put(a[i]);\
                        if ((_locks) & INODE_UPDATE_LOCK)                        \
                                mutex_unlock(&a[i]->ei_update_lock);        \
                }                                                        \
} while (0)

static inline struct bch_inode_info *file_bch_inode(struct file *file)
{
        return to_bch_ei(file_inode(file));
}

static inline bool inode_attr_changing(struct bch_inode_info *dir,
                                struct bch_inode_info *inode,
                                enum inode_opt_id id)
{
        return !(inode->ei_inode.bi_fields_set & (1 << id)) &&
                bch2_inode_opt_get(&dir->ei_inode, id) !=
                bch2_inode_opt_get(&inode->ei_inode, id);
}

static inline bool inode_attrs_changing(struct bch_inode_info *dir,
                                 struct bch_inode_info *inode)
{
        unsigned id;

        for (id = 0; id < Inode_opt_nr; id++)
                if (inode_attr_changing(dir, inode, id))
                        return true;

        return false;
}

struct bch_inode_unpacked;

#ifndef NO_BCACHEFS_FS

struct bch_inode_info *
__bch2_create(struct mnt_idmap *, struct bch_inode_info *,
              struct dentry *, umode_t, dev_t, subvol_inum, unsigned);

int bch2_inode_or_descendents_is_open(struct btree_trans *trans, struct bpos p);

int bch2_fs_quota_transfer(struct bch_fs *,
                           struct bch_inode_info *,
                           struct bch_qid,
                           unsigned,
                           enum quota_acct_mode);

static inline int bch2_set_projid(struct bch_fs *c,
                                  struct bch_inode_info *inode,
                                  u32 projid)
{
        struct bch_qid qid = inode->ei_qid;

        qid.q[QTYP_PRJ] = projid;

        return bch2_fs_quota_transfer(c, inode, qid,
                                      1 << QTYP_PRJ,
                                      KEY_TYPE_QUOTA_PREALLOC);
}

struct inode *bch2_vfs_inode_get(struct bch_fs *, subvol_inum);

/* returns 0 if we want to do the update, or error is passed up */
typedef int (*inode_set_fn)(struct btree_trans *,
                            struct bch_inode_info *,
                            struct bch_inode_unpacked *, void *);

void bch2_inode_update_after_write(struct btree_trans *,
                                   struct bch_inode_info *,
                                   struct bch_inode_unpacked *,
                                   unsigned);
int __must_check bch2_write_inode(struct bch_fs *, struct bch_inode_info *,
                                  inode_set_fn, void *, unsigned);

int bch2_setattr_nonsize(struct mnt_idmap *,
                         struct bch_inode_info *,
                         struct iattr *);
int __bch2_unlink(struct inode *, struct dentry *, bool);

void bch2_evict_subvolume_inodes(struct bch_fs *, snapshot_id_list *);

void bch2_fs_vfs_exit(struct bch_fs *);
int bch2_fs_vfs_init(struct bch_fs *);

void bch2_vfs_exit(void);
int bch2_vfs_init(void);

#else

#define bch2_inode_update_after_write(_trans, _inode, _inode_u, _fields)        ({ do {} while (0); })

static inline int bch2_inode_or_descendents_is_open(struct btree_trans *trans, struct bpos p) { return 0; }

static inline void bch2_evict_subvolume_inodes(struct bch_fs *c,
                                               snapshot_id_list *s) {}

static inline void bch2_fs_vfs_exit(struct bch_fs *c) {}
static inline int bch2_fs_vfs_init(struct bch_fs *c) { return 0; }

static inline void bch2_vfs_exit(void) {}
static inline int bch2_vfs_init(void) { return 0; }

#endif /* NO_BCACHEFS_FS */

#endif /* _BCACHEFS_FS_H */































































































































































































































































































































































































































































































































































   10 


   10 









    8 



    8 
    8 







    8 





















































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
/*
 * Copyright (c) 2016 Mellanox Technologies Ltd.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/security.h>
#include <linux/completion.h>
#include <linux/list.h>

#include <rdma/ib_verbs.h>
#include <rdma/ib_cache.h>
#include "core_priv.h"
#include "mad_priv.h"

static LIST_HEAD(mad_agent_list);
/* Lock to protect mad_agent_list */
static DEFINE_SPINLOCK(mad_agent_list_lock);

static struct pkey_index_qp_list *get_pkey_idx_qp_list(struct ib_port_pkey *pp)
{
        struct pkey_index_qp_list *pkey = NULL;
        struct pkey_index_qp_list *tmp_pkey;
        struct ib_device *dev = pp->sec->dev;

        spin_lock(&dev->port_data[pp->port_num].pkey_list_lock);
        list_for_each_entry (tmp_pkey, &dev->port_data[pp->port_num].pkey_list,
                             pkey_index_list) {
                if (tmp_pkey->pkey_index == pp->pkey_index) {
                        pkey = tmp_pkey;
                        break;
                }
        }
        spin_unlock(&dev->port_data[pp->port_num].pkey_list_lock);
        return pkey;
}

static int get_pkey_and_subnet_prefix(struct ib_port_pkey *pp,
                                      u16 *pkey,
                                      u64 *subnet_prefix)
{
        struct ib_device *dev = pp->sec->dev;
        int ret;

        ret = ib_get_cached_pkey(dev, pp->port_num, pp->pkey_index, pkey);
        if (ret)
                return ret;

        ib_get_cached_subnet_prefix(dev, pp->port_num, subnet_prefix);

        return ret;
}

static int enforce_qp_pkey_security(u16 pkey,
                                    u64 subnet_prefix,
                                    struct ib_qp_security *qp_sec)
{
        struct ib_qp_security *shared_qp_sec;
        int ret;

        ret = security_ib_pkey_access(qp_sec->security, subnet_prefix, pkey);
        if (ret)
                return ret;

        list_for_each_entry(shared_qp_sec,
                            &qp_sec->shared_qp_list,
                            shared_qp_list) {
                ret = security_ib_pkey_access(shared_qp_sec->security,
                                              subnet_prefix,
                                              pkey);
                if (ret)
                        return ret;
        }
        return 0;
}

/* The caller of this function must hold the QP security
 * mutex of the QP of the security structure in *pps.
 *
 * It takes separate ports_pkeys and security structure
 * because in some cases the pps will be for a new settings
 * or the pps will be for the real QP and security structure
 * will be for a shared QP.
 */
static int check_qp_port_pkey_settings(struct ib_ports_pkeys *pps,
                                       struct ib_qp_security *sec)
{
        u64 subnet_prefix;
        u16 pkey;
        int ret = 0;

        if (!pps)
                return 0;

        if (pps->main.state != IB_PORT_PKEY_NOT_VALID) {
                ret = get_pkey_and_subnet_prefix(&pps->main,
                                                 &pkey,
                                                 &subnet_prefix);
                if (ret)
                        return ret;

                ret = enforce_qp_pkey_security(pkey,
                                               subnet_prefix,
                                               sec);
                if (ret)
                        return ret;
        }

        if (pps->alt.state != IB_PORT_PKEY_NOT_VALID) {
                ret = get_pkey_and_subnet_prefix(&pps->alt,
                                                 &pkey,
                                                 &subnet_prefix);
                if (ret)
                        return ret;

                ret = enforce_qp_pkey_security(pkey,
                                               subnet_prefix,
                                               sec);
        }

        return ret;
}

/* The caller of this function must hold the QP security
 * mutex.
 */
static void qp_to_error(struct ib_qp_security *sec)
{
        struct ib_qp_security *shared_qp_sec;
        struct ib_qp_attr attr = {
                .qp_state = IB_QPS_ERR
        };
        struct ib_event event = {
                .event = IB_EVENT_QP_FATAL
        };

        /* If the QP is in the process of being destroyed
         * the qp pointer in the security structure is
         * undefined.  It cannot be modified now.
         */
        if (sec->destroying)
                return;

        ib_modify_qp(sec->qp,
                     &attr,
                     IB_QP_STATE);

        if (sec->qp->event_handler && sec->qp->qp_context) {
                event.element.qp = sec->qp;
                sec->qp->event_handler(&event,
                                       sec->qp->qp_context);
        }

        list_for_each_entry(shared_qp_sec,
                            &sec->shared_qp_list,
                            shared_qp_list) {
                struct ib_qp *qp = shared_qp_sec->qp;

                if (qp->event_handler && qp->qp_context) {
                        event.element.qp = qp;
                        event.device = qp->device;
                        qp->event_handler(&event,
                                          qp->qp_context);
                }
        }
}

static inline void check_pkey_qps(struct pkey_index_qp_list *pkey,
                                  struct ib_device *device,
                                  u32 port_num,
                                  u64 subnet_prefix)
{
        struct ib_port_pkey *pp, *tmp_pp;
        bool comp;
        LIST_HEAD(to_error_list);
        u16 pkey_val;

        if (!ib_get_cached_pkey(device,
                                port_num,
                                pkey->pkey_index,
                                &pkey_val)) {
                spin_lock(&pkey->qp_list_lock);
                list_for_each_entry(pp, &pkey->qp_list, qp_list) {
                        if (atomic_read(&pp->sec->error_list_count))
                                continue;

                        if (enforce_qp_pkey_security(pkey_val,
                                                     subnet_prefix,
                                                     pp->sec)) {
                                atomic_inc(&pp->sec->error_list_count);
                                list_add(&pp->to_error_list,
                                         &to_error_list);
                        }
                }
                spin_unlock(&pkey->qp_list_lock);
        }

        list_for_each_entry_safe(pp,
                                 tmp_pp,
                                 &to_error_list,
                                 to_error_list) {
                mutex_lock(&pp->sec->mutex);
                qp_to_error(pp->sec);
                list_del(&pp->to_error_list);
                atomic_dec(&pp->sec->error_list_count);
                comp = pp->sec->destroying;
                mutex_unlock(&pp->sec->mutex);

                if (comp)
                        complete(&pp->sec->error_complete);
        }
}

/* The caller of this function must hold the QP security
 * mutex.
 */
static int port_pkey_list_insert(struct ib_port_pkey *pp)
{
        struct pkey_index_qp_list *tmp_pkey;
        struct pkey_index_qp_list *pkey;
        struct ib_device *dev;
        u32 port_num = pp->port_num;
        int ret = 0;

        if (pp->state != IB_PORT_PKEY_VALID)
                return 0;

        dev = pp->sec->dev;

        pkey = get_pkey_idx_qp_list(pp);

        if (!pkey) {
                bool found = false;

                pkey = kzalloc(sizeof(*pkey), GFP_KERNEL);
                if (!pkey)
                        return -ENOMEM;

                spin_lock(&dev->port_data[port_num].pkey_list_lock);
                /* Check for the PKey again.  A racing process may
                 * have created it.
                 */
                list_for_each_entry(tmp_pkey,
                                    &dev->port_data[port_num].pkey_list,
                                    pkey_index_list) {
                        if (tmp_pkey->pkey_index == pp->pkey_index) {
                                kfree(pkey);
                                pkey = tmp_pkey;
                                found = true;
                                break;
                        }
                }

                if (!found) {
                        pkey->pkey_index = pp->pkey_index;
                        spin_lock_init(&pkey->qp_list_lock);
                        INIT_LIST_HEAD(&pkey->qp_list);
                        list_add(&pkey->pkey_index_list,
                                 &dev->port_data[port_num].pkey_list);
                }
                spin_unlock(&dev->port_data[port_num].pkey_list_lock);
        }

        spin_lock(&pkey->qp_list_lock);
        list_add(&pp->qp_list, &pkey->qp_list);
        spin_unlock(&pkey->qp_list_lock);

        pp->state = IB_PORT_PKEY_LISTED;

        return ret;
}

/* The caller of this function must hold the QP security
 * mutex.
 */
static void port_pkey_list_remove(struct ib_port_pkey *pp)
{
        struct pkey_index_qp_list *pkey;

        if (pp->state != IB_PORT_PKEY_LISTED)
                return;

        pkey = get_pkey_idx_qp_list(pp);

        spin_lock(&pkey->qp_list_lock);
        list_del(&pp->qp_list);
        spin_unlock(&pkey->qp_list_lock);

        /* The setting may still be valid, i.e. after
         * a destroy has failed for example.
         */
        pp->state = IB_PORT_PKEY_VALID;
}

static void destroy_qp_security(struct ib_qp_security *sec)
{
        security_ib_free_security(sec->security);
        kfree(sec->ports_pkeys);
        kfree(sec);
}

/* The caller of this function must hold the QP security
 * mutex.
 */
static struct ib_ports_pkeys *get_new_pps(const struct ib_qp *qp,
                                          const struct ib_qp_attr *qp_attr,
                                          int qp_attr_mask)
{
        struct ib_ports_pkeys *new_pps;
        struct ib_ports_pkeys *qp_pps = qp->qp_sec->ports_pkeys;

        new_pps = kzalloc(sizeof(*new_pps), GFP_KERNEL);
        if (!new_pps)
                return NULL;

        if (qp_attr_mask & IB_QP_PORT)
                new_pps->main.port_num = qp_attr->port_num;
        else if (qp_pps)
                new_pps->main.port_num = qp_pps->main.port_num;

        if (qp_attr_mask & IB_QP_PKEY_INDEX)
                new_pps->main.pkey_index = qp_attr->pkey_index;
        else if (qp_pps)
                new_pps->main.pkey_index = qp_pps->main.pkey_index;

        if (((qp_attr_mask & IB_QP_PKEY_INDEX) &&
             (qp_attr_mask & IB_QP_PORT)) ||
            (qp_pps && qp_pps->main.state != IB_PORT_PKEY_NOT_VALID))
                new_pps->main.state = IB_PORT_PKEY_VALID;

        if (qp_attr_mask & IB_QP_ALT_PATH) {
                new_pps->alt.port_num = qp_attr->alt_port_num;
                new_pps->alt.pkey_index = qp_attr->alt_pkey_index;
                new_pps->alt.state = IB_PORT_PKEY_VALID;
        } else if (qp_pps) {
                new_pps->alt.port_num = qp_pps->alt.port_num;
                new_pps->alt.pkey_index = qp_pps->alt.pkey_index;
                if (qp_pps->alt.state != IB_PORT_PKEY_NOT_VALID)
                        new_pps->alt.state = IB_PORT_PKEY_VALID;
        }

        new_pps->main.sec = qp->qp_sec;
        new_pps->alt.sec = qp->qp_sec;
        return new_pps;
}

int ib_open_shared_qp_security(struct ib_qp *qp, struct ib_device *dev)
{
        struct ib_qp *real_qp = qp->real_qp;
        int ret;

        ret = ib_create_qp_security(qp, dev);

        if (ret)
                return ret;

        if (!qp->qp_sec)
                return 0;

        mutex_lock(&real_qp->qp_sec->mutex);
        ret = check_qp_port_pkey_settings(real_qp->qp_sec->ports_pkeys,
                                          qp->qp_sec);

        if (ret)
                goto ret;

        if (qp != real_qp)
                list_add(&qp->qp_sec->shared_qp_list,
                         &real_qp->qp_sec->shared_qp_list);
ret:
        mutex_unlock(&real_qp->qp_sec->mutex);
        if (ret)
                destroy_qp_security(qp->qp_sec);

        return ret;
}

void ib_close_shared_qp_security(struct ib_qp_security *sec)
{
        struct ib_qp *real_qp = sec->qp->real_qp;

        mutex_lock(&real_qp->qp_sec->mutex);
        list_del(&sec->shared_qp_list);
        mutex_unlock(&real_qp->qp_sec->mutex);

        destroy_qp_security(sec);
}

int ib_create_qp_security(struct ib_qp *qp, struct ib_device *dev)
{
        unsigned int i;
        bool is_ib = false;
        int ret;

        rdma_for_each_port (dev, i) {
                is_ib = rdma_protocol_ib(dev, i);
                if (is_ib)
                        break;
        }

        /* If this isn't an IB device don't create the security context */
        if (!is_ib)
                return 0;

        qp->qp_sec = kzalloc(sizeof(*qp->qp_sec), GFP_KERNEL);
        if (!qp->qp_sec)
                return -ENOMEM;

        qp->qp_sec->qp = qp;
        qp->qp_sec->dev = dev;
        mutex_init(&qp->qp_sec->mutex);
        INIT_LIST_HEAD(&qp->qp_sec->shared_qp_list);
        atomic_set(&qp->qp_sec->error_list_count, 0);
        init_completion(&qp->qp_sec->error_complete);
        ret = security_ib_alloc_security(&qp->qp_sec->security);
        if (ret) {
                kfree(qp->qp_sec);
                qp->qp_sec = NULL;
        }

        return ret;
}
EXPORT_SYMBOL(ib_create_qp_security);

void ib_destroy_qp_security_begin(struct ib_qp_security *sec)
{
        /* Return if not IB */
        if (!sec)
                return;

        mutex_lock(&sec->mutex);

        /* Remove the QP from the lists so it won't get added to
         * a to_error_list during the destroy process.
         */
        if (sec->ports_pkeys) {
                port_pkey_list_remove(&sec->ports_pkeys->main);
                port_pkey_list_remove(&sec->ports_pkeys->alt);
        }

        /* If the QP is already in one or more of those lists
         * the destroying flag will ensure the to error flow
         * doesn't operate on an undefined QP.
         */
        sec->destroying = true;

        /* Record the error list count to know how many completions
         * to wait for.
         */
        sec->error_comps_pending = atomic_read(&sec->error_list_count);

        mutex_unlock(&sec->mutex);
}

void ib_destroy_qp_security_abort(struct ib_qp_security *sec)
{
        int ret;
        int i;

        /* Return if not IB */
        if (!sec)
                return;

        /* If a concurrent cache update is in progress this
         * QP security could be marked for an error state
         * transition.  Wait for this to complete.
         */
        for (i = 0; i < sec->error_comps_pending; i++)
                wait_for_completion(&sec->error_complete);

        mutex_lock(&sec->mutex);
        sec->destroying = false;

        /* Restore the position in the lists and verify
         * access is still allowed in case a cache update
         * occurred while attempting to destroy.
         *
         * Because these setting were listed already
         * and removed during ib_destroy_qp_security_begin
         * we know the pkey_index_qp_list for the PKey
         * already exists so port_pkey_list_insert won't fail.
         */
        if (sec->ports_pkeys) {
                port_pkey_list_insert(&sec->ports_pkeys->main);
                port_pkey_list_insert(&sec->ports_pkeys->alt);
        }

        ret = check_qp_port_pkey_settings(sec->ports_pkeys, sec);
        if (ret)
                qp_to_error(sec);

        mutex_unlock(&sec->mutex);
}

void ib_destroy_qp_security_end(struct ib_qp_security *sec)
{
        int i;

        /* Return if not IB */
        if (!sec)
                return;

        /* If a concurrent cache update is occurring we must
         * wait until this QP security structure is processed
         * in the QP to error flow before destroying it because
         * the to_error_list is in use.
         */
        for (i = 0; i < sec->error_comps_pending; i++)
                wait_for_completion(&sec->error_complete);

        destroy_qp_security(sec);
}

void ib_security_cache_change(struct ib_device *device,
                              u32 port_num,
                              u64 subnet_prefix)
{
        struct pkey_index_qp_list *pkey;

        list_for_each_entry (pkey, &device->port_data[port_num].pkey_list,
                             pkey_index_list) {
                check_pkey_qps(pkey,
                               device,
                               port_num,
                               subnet_prefix);
        }
}

void ib_security_release_port_pkey_list(struct ib_device *device)
{
        struct pkey_index_qp_list *pkey, *tmp_pkey;
        unsigned int i;

        rdma_for_each_port (device, i) {
                list_for_each_entry_safe(pkey,
                                         tmp_pkey,
                                         &device->port_data[i].pkey_list,
                                         pkey_index_list) {
                        list_del(&pkey->pkey_index_list);
                        kfree(pkey);
                }
        }
}

int ib_security_modify_qp(struct ib_qp *qp,
                          struct ib_qp_attr *qp_attr,
                          int qp_attr_mask,
                          struct ib_udata *udata)
{
        int ret = 0;
        struct ib_ports_pkeys *tmp_pps;
        struct ib_ports_pkeys *new_pps = NULL;
        struct ib_qp *real_qp = qp->real_qp;
        bool special_qp = (real_qp->qp_type == IB_QPT_SMI ||
                           real_qp->qp_type == IB_QPT_GSI ||
                           real_qp->qp_type >= IB_QPT_RESERVED1);
        bool pps_change = ((qp_attr_mask & (IB_QP_PKEY_INDEX | IB_QP_PORT)) ||
                           (qp_attr_mask & IB_QP_ALT_PATH));

        WARN_ONCE((qp_attr_mask & IB_QP_PORT &&
                   rdma_protocol_ib(real_qp->device, qp_attr->port_num) &&
                   !real_qp->qp_sec),
                   "%s: QP security is not initialized for IB QP: %u\n",
                   __func__, real_qp->qp_num);

        /* The port/pkey settings are maintained only for the real QP. Open
         * handles on the real QP will be in the shared_qp_list. When
         * enforcing security on the real QP all the shared QPs will be
         * checked as well.
         */

        if (pps_change && !special_qp && real_qp->qp_sec) {
                mutex_lock(&real_qp->qp_sec->mutex);
                new_pps = get_new_pps(real_qp,
                                      qp_attr,
                                      qp_attr_mask);
                if (!new_pps) {
                        mutex_unlock(&real_qp->qp_sec->mutex);
                        return -ENOMEM;
                }
                /* Add this QP to the lists for the new port
                 * and pkey settings before checking for permission
                 * in case there is a concurrent cache update
                 * occurring.  Walking the list for a cache change
                 * doesn't acquire the security mutex unless it's
                 * sending the QP to error.
                 */
                ret = port_pkey_list_insert(&new_pps->main);

                if (!ret)
                        ret = port_pkey_list_insert(&new_pps->alt);

                if (!ret)
                        ret = check_qp_port_pkey_settings(new_pps,
                                                          real_qp->qp_sec);
        }

        if (!ret)
                ret = real_qp->device->ops.modify_qp(real_qp,
                                                     qp_attr,
                                                     qp_attr_mask,
                                                     udata);

        if (new_pps) {
                /* Clean up the lists and free the appropriate
                 * ports_pkeys structure.
                 */
                if (ret) {
                        tmp_pps = new_pps;
                } else {
                        tmp_pps = real_qp->qp_sec->ports_pkeys;
                        real_qp->qp_sec->ports_pkeys = new_pps;
                }

                if (tmp_pps) {
                        port_pkey_list_remove(&tmp_pps->main);
                        port_pkey_list_remove(&tmp_pps->alt);
                }
                kfree(tmp_pps);
                mutex_unlock(&real_qp->qp_sec->mutex);
        }
        return ret;
}

static int ib_security_pkey_access(struct ib_device *dev,
                                   u32 port_num,
                                   u16 pkey_index,
                                   void *sec)
{
        u64 subnet_prefix;
        u16 pkey;
        int ret;

        if (!rdma_protocol_ib(dev, port_num))
                return 0;

        ret = ib_get_cached_pkey(dev, port_num, pkey_index, &pkey);
        if (ret)
                return ret;

        ib_get_cached_subnet_prefix(dev, port_num, &subnet_prefix);

        return security_ib_pkey_access(sec, subnet_prefix, pkey);
}

void ib_mad_agent_security_change(void)
{
        struct ib_mad_agent *ag;

        spin_lock(&mad_agent_list_lock);
        list_for_each_entry(ag,
                            &mad_agent_list,
                            mad_agent_sec_list)
                WRITE_ONCE(ag->smp_allowed,
                           !security_ib_endport_manage_subnet(ag->security,
                                dev_name(&ag->device->dev), ag->port_num));
        spin_unlock(&mad_agent_list_lock);
}

int ib_mad_agent_security_setup(struct ib_mad_agent *agent,
                                enum ib_qp_type qp_type)
{
        int ret;

        if (!rdma_protocol_ib(agent->device, agent->port_num))
                return 0;

        INIT_LIST_HEAD(&agent->mad_agent_sec_list);

        ret = security_ib_alloc_security(&agent->security);
        if (ret)
                return ret;

        if (qp_type != IB_QPT_SMI)
                return 0;

        spin_lock(&mad_agent_list_lock);
        ret = security_ib_endport_manage_subnet(agent->security,
                                                dev_name(&agent->device->dev),
                                                agent->port_num);
        if (ret)
                goto free_security;

        WRITE_ONCE(agent->smp_allowed, true);
        list_add(&agent->mad_agent_sec_list, &mad_agent_list);
        spin_unlock(&mad_agent_list_lock);
        return 0;

free_security:
        spin_unlock(&mad_agent_list_lock);
        security_ib_free_security(agent->security);
        return ret;
}

void ib_mad_agent_security_cleanup(struct ib_mad_agent *agent)
{
        if (!rdma_protocol_ib(agent->device, agent->port_num))
                return;

        if (agent->qp->qp_type == IB_QPT_SMI) {
                spin_lock(&mad_agent_list_lock);
                list_del(&agent->mad_agent_sec_list);
                spin_unlock(&mad_agent_list_lock);
        }

        security_ib_free_security(agent->security);
}

int ib_mad_enforce_security(struct ib_mad_agent_private *map, u16 pkey_index)
{
        if (!rdma_protocol_ib(map->agent.device, map->agent.port_num))
                return 0;

        if (map->agent.qp->qp_type == IB_QPT_SMI) {
                if (!READ_ONCE(map->agent.smp_allowed))
                        return -EACCES;
                return 0;
        }

        return ib_security_pkey_access(map->agent.device,
                                       map->agent.port_num,
                                       pkey_index,
                                       map->agent.security);
}


























































































 2898 






  504 





































 4588 






 3804 






 3804 































































































    1 































































































  433 






  433 
















































  105 








































  149 





























































  514 

































  111 
























































































































































































































































  523 






  523 

































    4 





    1 





 3136 










  828 




  356 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM writeback

#if !defined(_TRACE_WRITEBACK_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_WRITEBACK_H

#include <linux/tracepoint.h>
#include <linux/backing-dev.h>
#include <linux/writeback.h>

#define show_inode_state(state)                                        \
        __print_flags(state, "|",                                \
                {I_DIRTY_SYNC,                "I_DIRTY_SYNC"},        \
                {I_DIRTY_DATASYNC,        "I_DIRTY_DATASYNC"},        \
                {I_DIRTY_PAGES,                "I_DIRTY_PAGES"},        \
                {I_NEW,                        "I_NEW"},                \
                {I_WILL_FREE,                "I_WILL_FREE"},                \
                {I_FREEING,                "I_FREEING"},                \
                {I_CLEAR,                "I_CLEAR"},                \
                {I_SYNC,                "I_SYNC"},                \
                {I_DIRTY_TIME,                "I_DIRTY_TIME"},        \
                {I_REFERENCED,                "I_REFERENCED"},        \
                {I_LINKABLE,                "I_LINKABLE"},                \
                {I_WB_SWITCH,                "I_WB_SWITCH"},                \
                {I_OVL_INUSE,                "I_OVL_INUSE"},                \
                {I_CREATING,                "I_CREATING"},                \
                {I_DONTCACHE,                "I_DONTCACHE"},                \
                {I_SYNC_QUEUED,                "I_SYNC_QUEUED"},        \
                {I_PINNING_NETFS_WB,        "I_PINNING_NETFS_WB"},        \
                {I_LRU_ISOLATING,        "I_LRU_ISOLATING"}        \
        )

/* enums need to be exported to user space */
#undef EM
#undef EMe
#define EM(a,b)         TRACE_DEFINE_ENUM(a);
#define EMe(a,b)        TRACE_DEFINE_ENUM(a);

#define WB_WORK_REASON                                                        \
        EM( WB_REASON_BACKGROUND,                "background")                \
        EM( WB_REASON_VMSCAN,                        "vmscan")                \
        EM( WB_REASON_SYNC,                        "sync")                        \
        EM( WB_REASON_PERIODIC,                        "periodic")                \
        EM( WB_REASON_LAPTOP_TIMER,                "laptop_timer")                \
        EM( WB_REASON_FS_FREE_SPACE,                "fs_free_space")        \
        EM( WB_REASON_FORKER_THREAD,                "forker_thread")        \
        EMe(WB_REASON_FOREIGN_FLUSH,                "foreign_flush")

WB_WORK_REASON

/*
 * Now redefine the EM() and EMe() macros to map the enums to the strings
 * that will be printed in the output.
 */
#undef EM
#undef EMe
#define EM(a,b)                { a, b },
#define EMe(a,b)        { a, b }

struct wb_writeback_work;

DECLARE_EVENT_CLASS(writeback_folio_template,

        TP_PROTO(struct folio *folio, struct address_space *mapping),

        TP_ARGS(folio, mapping),

        TP_STRUCT__entry (
                __array(char, name, 32)
                __field(ino_t, ino)
                __field(pgoff_t, index)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name,
                            bdi_dev_name(mapping ? inode_to_bdi(mapping->host) :
                                         NULL), 32);
                __entry->ino = (mapping && mapping->host) ? mapping->host->i_ino : 0;
                __entry->index = folio->index;
        ),

        TP_printk("bdi %s: ino=%lu index=%lu",
                __entry->name,
                (unsigned long)__entry->ino,
                __entry->index
        )
);

DEFINE_EVENT(writeback_folio_template, writeback_dirty_folio,

        TP_PROTO(struct folio *folio, struct address_space *mapping),

        TP_ARGS(folio, mapping)
);

DEFINE_EVENT(writeback_folio_template, folio_wait_writeback,

        TP_PROTO(struct folio *folio, struct address_space *mapping),

        TP_ARGS(folio, mapping)
);

DECLARE_EVENT_CLASS(writeback_dirty_inode_template,

        TP_PROTO(struct inode *inode, int flags),

        TP_ARGS(inode, flags),

        TP_STRUCT__entry (
                __array(char, name, 32)
                __field(ino_t, ino)
                __field(unsigned long, state)
                __field(unsigned long, flags)
        ),

        TP_fast_assign(
                struct backing_dev_info *bdi = inode_to_bdi(inode);

                /* may be called for files on pseudo FSes w/ unregistered bdi */
                strscpy_pad(__entry->name, bdi_dev_name(bdi), 32);
                __entry->ino                = inode->i_ino;
                __entry->state                = inode->i_state;
                __entry->flags                = flags;
        ),

        TP_printk("bdi %s: ino=%lu state=%s flags=%s",
                __entry->name,
                (unsigned long)__entry->ino,
                show_inode_state(__entry->state),
                show_inode_state(__entry->flags)
        )
);

DEFINE_EVENT(writeback_dirty_inode_template, writeback_mark_inode_dirty,

        TP_PROTO(struct inode *inode, int flags),

        TP_ARGS(inode, flags)
);

DEFINE_EVENT(writeback_dirty_inode_template, writeback_dirty_inode_start,

        TP_PROTO(struct inode *inode, int flags),

        TP_ARGS(inode, flags)
);

DEFINE_EVENT(writeback_dirty_inode_template, writeback_dirty_inode,

        TP_PROTO(struct inode *inode, int flags),

        TP_ARGS(inode, flags)
);

#ifdef CREATE_TRACE_POINTS
#ifdef CONFIG_CGROUP_WRITEBACK

static inline ino_t __trace_wb_assign_cgroup(struct bdi_writeback *wb)
{
        return cgroup_ino(wb->memcg_css->cgroup);
}

static inline ino_t __trace_wbc_assign_cgroup(struct writeback_control *wbc)
{
        if (wbc->wb)
                return __trace_wb_assign_cgroup(wbc->wb);
        else
                return 1;
}
#else        /* CONFIG_CGROUP_WRITEBACK */

static inline ino_t __trace_wb_assign_cgroup(struct bdi_writeback *wb)
{
        return 1;
}

static inline ino_t __trace_wbc_assign_cgroup(struct writeback_control *wbc)
{
        return 1;
}

#endif        /* CONFIG_CGROUP_WRITEBACK */
#endif        /* CREATE_TRACE_POINTS */

#ifdef CONFIG_CGROUP_WRITEBACK
TRACE_EVENT(inode_foreign_history,

        TP_PROTO(struct inode *inode, struct writeback_control *wbc,
                 unsigned int history),

        TP_ARGS(inode, wbc, history),

        TP_STRUCT__entry(
                __array(char,                name, 32)
                __field(ino_t,                ino)
                __field(ino_t,                cgroup_ino)
                __field(unsigned int,        history)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(inode_to_bdi(inode)), 32);
                __entry->ino                = inode->i_ino;
                __entry->cgroup_ino        = __trace_wbc_assign_cgroup(wbc);
                __entry->history        = history;
        ),

        TP_printk("bdi %s: ino=%lu cgroup_ino=%lu history=0x%x",
                __entry->name,
                (unsigned long)__entry->ino,
                (unsigned long)__entry->cgroup_ino,
                __entry->history
        )
);

TRACE_EVENT(inode_switch_wbs,

        TP_PROTO(struct inode *inode, struct bdi_writeback *old_wb,
                 struct bdi_writeback *new_wb),

        TP_ARGS(inode, old_wb, new_wb),

        TP_STRUCT__entry(
                __array(char,                name, 32)
                __field(ino_t,                ino)
                __field(ino_t,                old_cgroup_ino)
                __field(ino_t,                new_cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(old_wb->bdi), 32);
                __entry->ino                = inode->i_ino;
                __entry->old_cgroup_ino        = __trace_wb_assign_cgroup(old_wb);
                __entry->new_cgroup_ino        = __trace_wb_assign_cgroup(new_wb);
        ),

        TP_printk("bdi %s: ino=%lu old_cgroup_ino=%lu new_cgroup_ino=%lu",
                __entry->name,
                (unsigned long)__entry->ino,
                (unsigned long)__entry->old_cgroup_ino,
                (unsigned long)__entry->new_cgroup_ino
        )
);

TRACE_EVENT(track_foreign_dirty,

        TP_PROTO(struct folio *folio, struct bdi_writeback *wb),

        TP_ARGS(folio, wb),

        TP_STRUCT__entry(
                __array(char,                name, 32)
                __field(u64,                bdi_id)
                __field(ino_t,                ino)
                __field(unsigned int,        memcg_id)
                __field(ino_t,                cgroup_ino)
                __field(ino_t,                page_cgroup_ino)
        ),

        TP_fast_assign(
                struct address_space *mapping = folio_mapping(folio);
                struct inode *inode = mapping ? mapping->host : NULL;

                strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
                __entry->bdi_id                = wb->bdi->id;
                __entry->ino                = inode ? inode->i_ino : 0;
                __entry->memcg_id        = wb->memcg_css->id;
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(wb);
                __entry->page_cgroup_ino = cgroup_ino(folio_memcg(folio)->css.cgroup);
        ),

        TP_printk("bdi %s[%llu]: ino=%lu memcg_id=%u cgroup_ino=%lu page_cgroup_ino=%lu",
                __entry->name,
                __entry->bdi_id,
                (unsigned long)__entry->ino,
                __entry->memcg_id,
                (unsigned long)__entry->cgroup_ino,
                (unsigned long)__entry->page_cgroup_ino
        )
);

TRACE_EVENT(flush_foreign,

        TP_PROTO(struct bdi_writeback *wb, unsigned int frn_bdi_id,
                 unsigned int frn_memcg_id),

        TP_ARGS(wb, frn_bdi_id, frn_memcg_id),

        TP_STRUCT__entry(
                __array(char,                name, 32)
                __field(ino_t,                cgroup_ino)
                __field(unsigned int,        frn_bdi_id)
                __field(unsigned int,        frn_memcg_id)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(wb);
                __entry->frn_bdi_id        = frn_bdi_id;
                __entry->frn_memcg_id        = frn_memcg_id;
        ),

        TP_printk("bdi %s: cgroup_ino=%lu frn_bdi_id=%u frn_memcg_id=%u",
                __entry->name,
                (unsigned long)__entry->cgroup_ino,
                __entry->frn_bdi_id,
                __entry->frn_memcg_id
        )
);
#endif

DECLARE_EVENT_CLASS(writeback_write_inode_template,

        TP_PROTO(struct inode *inode, struct writeback_control *wbc),

        TP_ARGS(inode, wbc),

        TP_STRUCT__entry (
                __array(char, name, 32)
                __field(ino_t, ino)
                __field(int, sync_mode)
                __field(ino_t, cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name,
                            bdi_dev_name(inode_to_bdi(inode)), 32);
                __entry->ino                = inode->i_ino;
                __entry->sync_mode        = wbc->sync_mode;
                __entry->cgroup_ino        = __trace_wbc_assign_cgroup(wbc);
        ),

        TP_printk("bdi %s: ino=%lu sync_mode=%d cgroup_ino=%lu",
                __entry->name,
                (unsigned long)__entry->ino,
                __entry->sync_mode,
                (unsigned long)__entry->cgroup_ino
        )
);

DEFINE_EVENT(writeback_write_inode_template, writeback_write_inode_start,

        TP_PROTO(struct inode *inode, struct writeback_control *wbc),

        TP_ARGS(inode, wbc)
);

DEFINE_EVENT(writeback_write_inode_template, writeback_write_inode,

        TP_PROTO(struct inode *inode, struct writeback_control *wbc),

        TP_ARGS(inode, wbc)
);

DECLARE_EVENT_CLASS(writeback_work_class,
        TP_PROTO(struct bdi_writeback *wb, struct wb_writeback_work *work),
        TP_ARGS(wb, work),
        TP_STRUCT__entry(
                __array(char, name, 32)
                __field(long, nr_pages)
                __field(dev_t, sb_dev)
                __field(int, sync_mode)
                __field(int, for_kupdate)
                __field(int, range_cyclic)
                __field(int, for_background)
                __field(int, reason)
                __field(ino_t, cgroup_ino)
        ),
        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
                __entry->nr_pages = work->nr_pages;
                __entry->sb_dev = work->sb ? work->sb->s_dev : 0;
                __entry->sync_mode = work->sync_mode;
                __entry->for_kupdate = work->for_kupdate;
                __entry->range_cyclic = work->range_cyclic;
                __entry->for_background        = work->for_background;
                __entry->reason = work->reason;
                __entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
        ),
        TP_printk("bdi %s: sb_dev %d:%d nr_pages=%ld sync_mode=%d "
                  "kupdate=%d range_cyclic=%d background=%d reason=%s cgroup_ino=%lu",
                  __entry->name,
                  MAJOR(__entry->sb_dev), MINOR(__entry->sb_dev),
                  __entry->nr_pages,
                  __entry->sync_mode,
                  __entry->for_kupdate,
                  __entry->range_cyclic,
                  __entry->for_background,
                  __print_symbolic(__entry->reason, WB_WORK_REASON),
                  (unsigned long)__entry->cgroup_ino
        )
);
#define DEFINE_WRITEBACK_WORK_EVENT(name) \
DEFINE_EVENT(writeback_work_class, name, \
        TP_PROTO(struct bdi_writeback *wb, struct wb_writeback_work *work), \
        TP_ARGS(wb, work))
DEFINE_WRITEBACK_WORK_EVENT(writeback_queue);
DEFINE_WRITEBACK_WORK_EVENT(writeback_exec);
DEFINE_WRITEBACK_WORK_EVENT(writeback_start);
DEFINE_WRITEBACK_WORK_EVENT(writeback_written);
DEFINE_WRITEBACK_WORK_EVENT(writeback_wait);

TRACE_EVENT(writeback_pages_written,
        TP_PROTO(long pages_written),
        TP_ARGS(pages_written),
        TP_STRUCT__entry(
                __field(long,                pages)
        ),
        TP_fast_assign(
                __entry->pages                = pages_written;
        ),
        TP_printk("%ld", __entry->pages)
);

DECLARE_EVENT_CLASS(writeback_class,
        TP_PROTO(struct bdi_writeback *wb),
        TP_ARGS(wb),
        TP_STRUCT__entry(
                __array(char, name, 32)
                __field(ino_t, cgroup_ino)
        ),
        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
                __entry->cgroup_ino = __trace_wb_assign_cgroup(wb);
        ),
        TP_printk("bdi %s: cgroup_ino=%lu",
                  __entry->name,
                  (unsigned long)__entry->cgroup_ino
        )
);
#define DEFINE_WRITEBACK_EVENT(name) \
DEFINE_EVENT(writeback_class, name, \
        TP_PROTO(struct bdi_writeback *wb), \
        TP_ARGS(wb))

DEFINE_WRITEBACK_EVENT(writeback_wake_background);

TRACE_EVENT(writeback_bdi_register,
        TP_PROTO(struct backing_dev_info *bdi),
        TP_ARGS(bdi),
        TP_STRUCT__entry(
                __array(char, name, 32)
        ),
        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(bdi), 32);
        ),
        TP_printk("bdi %s",
                __entry->name
        )
);

DECLARE_EVENT_CLASS(wbc_class,
        TP_PROTO(struct writeback_control *wbc, struct backing_dev_info *bdi),
        TP_ARGS(wbc, bdi),
        TP_STRUCT__entry(
                __array(char, name, 32)
                __field(long, nr_to_write)
                __field(long, pages_skipped)
                __field(int, sync_mode)
                __field(int, for_kupdate)
                __field(int, for_background)
                __field(int, range_cyclic)
                __field(long, range_start)
                __field(long, range_end)
                __field(ino_t, cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(bdi), 32);
                __entry->nr_to_write        = wbc->nr_to_write;
                __entry->pages_skipped        = wbc->pages_skipped;
                __entry->sync_mode        = wbc->sync_mode;
                __entry->for_kupdate        = wbc->for_kupdate;
                __entry->for_background        = wbc->for_background;
                __entry->range_cyclic        = wbc->range_cyclic;
                __entry->range_start        = (long)wbc->range_start;
                __entry->range_end        = (long)wbc->range_end;
                __entry->cgroup_ino        = __trace_wbc_assign_cgroup(wbc);
        ),

        TP_printk("bdi %s: towrt=%ld skip=%ld mode=%d kupd=%d bgrd=%d "
                "cyclic=%d start=0x%lx end=0x%lx cgroup_ino=%lu",
                __entry->name,
                __entry->nr_to_write,
                __entry->pages_skipped,
                __entry->sync_mode,
                __entry->for_kupdate,
                __entry->for_background,
                __entry->range_cyclic,
                __entry->range_start,
                __entry->range_end,
                (unsigned long)__entry->cgroup_ino
        )
)

#define DEFINE_WBC_EVENT(name) \
DEFINE_EVENT(wbc_class, name, \
        TP_PROTO(struct writeback_control *wbc, struct backing_dev_info *bdi), \
        TP_ARGS(wbc, bdi))
DEFINE_WBC_EVENT(wbc_writepage);

TRACE_EVENT(writeback_queue_io,
        TP_PROTO(struct bdi_writeback *wb,
                 struct wb_writeback_work *work,
                 unsigned long dirtied_before,
                 int moved),
        TP_ARGS(wb, work, dirtied_before, moved),
        TP_STRUCT__entry(
                __array(char,                name, 32)
                __field(unsigned long,        older)
                __field(long,                age)
                __field(int,                moved)
                __field(int,                reason)
                __field(ino_t,                cgroup_ino)
        ),
        TP_fast_assign(
                strscpy_pad(__entry->name, bdi_dev_name(wb->bdi), 32);
                __entry->older        = dirtied_before;
                __entry->age        = (jiffies - dirtied_before) * 1000 / HZ;
                __entry->moved        = moved;
                __entry->reason        = work->reason;
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(wb);
        ),
        TP_printk("bdi %s: older=%lu age=%ld enqueue=%d reason=%s cgroup_ino=%lu",
                __entry->name,
                __entry->older,        /* dirtied_before in jiffies */
                __entry->age,        /* dirtied_before in relative milliseconds */
                __entry->moved,
                __print_symbolic(__entry->reason, WB_WORK_REASON),
                (unsigned long)__entry->cgroup_ino
        )
);

TRACE_EVENT(global_dirty_state,

        TP_PROTO(unsigned long background_thresh,
                 unsigned long dirty_thresh
        ),

        TP_ARGS(background_thresh,
                dirty_thresh
        ),

        TP_STRUCT__entry(
                __field(unsigned long,        nr_dirty)
                __field(unsigned long,        nr_writeback)
                __field(unsigned long,        background_thresh)
                __field(unsigned long,        dirty_thresh)
                __field(unsigned long,        dirty_limit)
                __field(unsigned long,        nr_dirtied)
                __field(unsigned long,        nr_written)
        ),

        TP_fast_assign(
                __entry->nr_dirty        = global_node_page_state(NR_FILE_DIRTY);
                __entry->nr_writeback        = global_node_page_state(NR_WRITEBACK);
                __entry->nr_dirtied        = global_node_page_state(NR_DIRTIED);
                __entry->nr_written        = global_node_page_state(NR_WRITTEN);
                __entry->background_thresh = background_thresh;
                __entry->dirty_thresh        = dirty_thresh;
                __entry->dirty_limit        = global_wb_domain.dirty_limit;
        ),

        TP_printk("dirty=%lu writeback=%lu "
                  "bg_thresh=%lu thresh=%lu limit=%lu "
                  "dirtied=%lu written=%lu",
                  __entry->nr_dirty,
                  __entry->nr_writeback,
                  __entry->background_thresh,
                  __entry->dirty_thresh,
                  __entry->dirty_limit,
                  __entry->nr_dirtied,
                  __entry->nr_written
        )
);

#define KBps(x)                        ((x) << (PAGE_SHIFT - 10))

TRACE_EVENT(bdi_dirty_ratelimit,

        TP_PROTO(struct bdi_writeback *wb,
                 unsigned long dirty_rate,
                 unsigned long task_ratelimit),

        TP_ARGS(wb, dirty_rate, task_ratelimit),

        TP_STRUCT__entry(
                __array(char,                bdi, 32)
                __field(unsigned long,        write_bw)
                __field(unsigned long,        avg_write_bw)
                __field(unsigned long,        dirty_rate)
                __field(unsigned long,        dirty_ratelimit)
                __field(unsigned long,        task_ratelimit)
                __field(unsigned long,        balanced_dirty_ratelimit)
                __field(ino_t,                cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->bdi, bdi_dev_name(wb->bdi), 32);
                __entry->write_bw        = KBps(wb->write_bandwidth);
                __entry->avg_write_bw        = KBps(wb->avg_write_bandwidth);
                __entry->dirty_rate        = KBps(dirty_rate);
                __entry->dirty_ratelimit = KBps(wb->dirty_ratelimit);
                __entry->task_ratelimit        = KBps(task_ratelimit);
                __entry->balanced_dirty_ratelimit =
                                        KBps(wb->balanced_dirty_ratelimit);
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(wb);
        ),

        TP_printk("bdi %s: "
                  "write_bw=%lu awrite_bw=%lu dirty_rate=%lu "
                  "dirty_ratelimit=%lu task_ratelimit=%lu "
                  "balanced_dirty_ratelimit=%lu cgroup_ino=%lu",
                  __entry->bdi,
                  __entry->write_bw,                /* write bandwidth */
                  __entry->avg_write_bw,        /* avg write bandwidth */
                  __entry->dirty_rate,                /* bdi dirty rate */
                  __entry->dirty_ratelimit,        /* base ratelimit */
                  __entry->task_ratelimit, /* ratelimit with position control */
                  __entry->balanced_dirty_ratelimit, /* the balanced ratelimit */
                  (unsigned long)__entry->cgroup_ino
        )
);

TRACE_EVENT(balance_dirty_pages,

        TP_PROTO(struct bdi_writeback *wb,
                 struct dirty_throttle_control *dtc,
                 unsigned long dirty_ratelimit,
                 unsigned long task_ratelimit,
                 unsigned long dirtied,
                 unsigned long period,
                 long pause,
                 unsigned long start_time),

        TP_ARGS(wb, dtc,
                dirty_ratelimit, task_ratelimit,
                dirtied, period, pause, start_time),

        TP_STRUCT__entry(
                __array(         char,        bdi, 32)
                __field(unsigned long,        limit)
                __field(unsigned long,        setpoint)
                __field(unsigned long,        dirty)
                __field(unsigned long,        wb_setpoint)
                __field(unsigned long,        wb_dirty)
                __field(unsigned long,        dirty_ratelimit)
                __field(unsigned long,        task_ratelimit)
                __field(unsigned int,        dirtied)
                __field(unsigned int,        dirtied_pause)
                __field(unsigned long,        paused)
                __field(         long,        pause)
                __field(unsigned long,        period)
                __field(         long,        think)
                __field(ino_t,                cgroup_ino)
        ),

        TP_fast_assign(
                unsigned long freerun = (dtc->thresh + dtc->bg_thresh) / 2;
                strscpy_pad(__entry->bdi, bdi_dev_name(wb->bdi), 32);

                __entry->limit                = dtc->limit;
                __entry->setpoint        = (dtc->limit + freerun) / 2;
                __entry->dirty                = dtc->dirty;
                __entry->wb_setpoint        = __entry->setpoint *
                                                dtc->wb_thresh / (dtc->thresh + 1);
                __entry->wb_dirty        = dtc->wb_dirty;
                __entry->dirty_ratelimit = KBps(dirty_ratelimit);
                __entry->task_ratelimit        = KBps(task_ratelimit);
                __entry->dirtied        = dirtied;
                __entry->dirtied_pause        = current->nr_dirtied_pause;
                __entry->think                = current->dirty_paused_when == 0 ? 0 :
                         (long)(jiffies - current->dirty_paused_when) * 1000/HZ;
                __entry->period                = period * 1000 / HZ;
                __entry->pause                = pause * 1000 / HZ;
                __entry->paused                = (jiffies - start_time) * 1000 / HZ;
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(wb);
        ),


        TP_printk("bdi %s: "
                  "limit=%lu setpoint=%lu dirty=%lu "
                  "wb_setpoint=%lu wb_dirty=%lu "
                  "dirty_ratelimit=%lu task_ratelimit=%lu "
                  "dirtied=%u dirtied_pause=%u "
                  "paused=%lu pause=%ld period=%lu think=%ld cgroup_ino=%lu",
                  __entry->bdi,
                  __entry->limit,
                  __entry->setpoint,
                  __entry->dirty,
                  __entry->wb_setpoint,
                  __entry->wb_dirty,
                  __entry->dirty_ratelimit,
                  __entry->task_ratelimit,
                  __entry->dirtied,
                  __entry->dirtied_pause,
                  __entry->paused,        /* ms */
                  __entry->pause,        /* ms */
                  __entry->period,        /* ms */
                  __entry->think,        /* ms */
                  (unsigned long)__entry->cgroup_ino
          )
);

TRACE_EVENT(writeback_sb_inodes_requeue,

        TP_PROTO(struct inode *inode),
        TP_ARGS(inode),

        TP_STRUCT__entry(
                __array(char, name, 32)
                __field(ino_t, ino)
                __field(unsigned long, state)
                __field(unsigned long, dirtied_when)
                __field(ino_t, cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name,
                            bdi_dev_name(inode_to_bdi(inode)), 32);
                __entry->ino                = inode->i_ino;
                __entry->state                = inode->i_state;
                __entry->dirtied_when        = inode->dirtied_when;
                __entry->cgroup_ino        = __trace_wb_assign_cgroup(inode_to_wb(inode));
        ),

        TP_printk("bdi %s: ino=%lu state=%s dirtied_when=%lu age=%lu cgroup_ino=%lu",
                  __entry->name,
                  (unsigned long)__entry->ino,
                  show_inode_state(__entry->state),
                  __entry->dirtied_when,
                  (jiffies - __entry->dirtied_when) / HZ,
                  (unsigned long)__entry->cgroup_ino
        )
);

DECLARE_EVENT_CLASS(writeback_single_inode_template,

        TP_PROTO(struct inode *inode,
                 struct writeback_control *wbc,
                 unsigned long nr_to_write
        ),

        TP_ARGS(inode, wbc, nr_to_write),

        TP_STRUCT__entry(
                __array(char, name, 32)
                __field(ino_t, ino)
                __field(unsigned long, state)
                __field(unsigned long, dirtied_when)
                __field(unsigned long, writeback_index)
                __field(long, nr_to_write)
                __field(unsigned long, wrote)
                __field(ino_t, cgroup_ino)
        ),

        TP_fast_assign(
                strscpy_pad(__entry->name,
                            bdi_dev_name(inode_to_bdi(inode)), 32);
                __entry->ino                = inode->i_ino;
                __entry->state                = inode->i_state;
                __entry->dirtied_when        = inode->dirtied_when;
                __entry->writeback_index = inode->i_mapping->writeback_index;
                __entry->nr_to_write        = nr_to_write;
                __entry->wrote                = nr_to_write - wbc->nr_to_write;
                __entry->cgroup_ino        = __trace_wbc_assign_cgroup(wbc);
        ),

        TP_printk("bdi %s: ino=%lu state=%s dirtied_when=%lu age=%lu "
                  "index=%lu to_write=%ld wrote=%lu cgroup_ino=%lu",
                  __entry->name,
                  (unsigned long)__entry->ino,
                  show_inode_state(__entry->state),
                  __entry->dirtied_when,
                  (jiffies - __entry->dirtied_when) / HZ,
                  __entry->writeback_index,
                  __entry->nr_to_write,
                  __entry->wrote,
                  (unsigned long)__entry->cgroup_ino
        )
);

DEFINE_EVENT(writeback_single_inode_template, writeback_single_inode_start,
        TP_PROTO(struct inode *inode,
                 struct writeback_control *wbc,
                 unsigned long nr_to_write),
        TP_ARGS(inode, wbc, nr_to_write)
);

DEFINE_EVENT(writeback_single_inode_template, writeback_single_inode,
        TP_PROTO(struct inode *inode,
                 struct writeback_control *wbc,
                 unsigned long nr_to_write),
        TP_ARGS(inode, wbc, nr_to_write)
);

DECLARE_EVENT_CLASS(writeback_inode_template,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode),

        TP_STRUCT__entry(
                __field(        dev_t,        dev                        )
                __field(        ino_t,        ino                        )
                __field(unsigned long,        state                        )
                __field(        __u16, mode                        )
                __field(unsigned long, dirtied_when                )
        ),

        TP_fast_assign(
                __entry->dev        = inode->i_sb->s_dev;
                __entry->ino        = inode->i_ino;
                __entry->state        = inode->i_state;
                __entry->mode        = inode->i_mode;
                __entry->dirtied_when = inode->dirtied_when;
        ),

        TP_printk("dev %d,%d ino %lu dirtied %lu state %s mode 0%o",
                  MAJOR(__entry->dev), MINOR(__entry->dev),
                  (unsigned long)__entry->ino, __entry->dirtied_when,
                  show_inode_state(__entry->state), __entry->mode)
);

DEFINE_EVENT(writeback_inode_template, writeback_lazytime,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode)
);

DEFINE_EVENT(writeback_inode_template, writeback_lazytime_iput,
        TP_PROTO(struct inode *inode),

        TP_ARGS(inode)
);

DEFINE_EVENT(writeback_inode_template, writeback_dirty_inode_enqueue,

        TP_PROTO(struct inode *inode),

        TP_ARGS(inode)
);

/*
 * Inode writeback list tracking.
 */

DEFINE_EVENT(writeback_inode_template, sb_mark_inode_writeback,
        TP_PROTO(struct inode *inode),
        TP_ARGS(inode)
);

DEFINE_EVENT(writeback_inode_template, sb_clear_inode_writeback,
        TP_PROTO(struct inode *inode),
        TP_ARGS(inode)
);

#endif /* _TRACE_WRITEBACK_H */

/* This part must be outside protection */
#include <trace/define_trace.h>
























































































































































































































































   10 






















































































































































   10 
 23830 






























































 11972 


























































   99 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
/* SPDX-License-Identifier: GPL-2.0+ */
/*
 * Sleepable Read-Copy Update mechanism for mutual exclusion
 *
 * Copyright (C) IBM Corporation, 2006
 * Copyright (C) Fujitsu, 2012
 *
 * Author: Paul McKenney <paulmck@linux.ibm.com>
 *           Lai Jiangshan <laijs@cn.fujitsu.com>
 *
 * For detailed explanation of Read-Copy Update mechanism see -
 *                Documentation/RCU/ *.txt
 *
 */

#ifndef _LINUX_SRCU_H
#define _LINUX_SRCU_H

#include <linux/mutex.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/rcu_segcblist.h>

struct srcu_struct;

#ifdef CONFIG_DEBUG_LOCK_ALLOC

int __init_srcu_struct(struct srcu_struct *ssp, const char *name,
                       struct lock_class_key *key);

#define init_srcu_struct(ssp) \
({ \
        static struct lock_class_key __srcu_key; \
        \
        __init_srcu_struct((ssp), #ssp, &__srcu_key); \
})

#define __SRCU_DEP_MAP_INIT(srcu_name)        .dep_map = { .name = #srcu_name },
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

int init_srcu_struct(struct srcu_struct *ssp);

#define __SRCU_DEP_MAP_INIT(srcu_name)
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */

/* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */
#define SRCU_READ_FLAVOR_NORMAL        0x1                // srcu_read_lock().
#define SRCU_READ_FLAVOR_NMI        0x2                // srcu_read_lock_nmisafe().
#define SRCU_READ_FLAVOR_LITE        0x4                // srcu_read_lock_lite().
#define SRCU_READ_FLAVOR_FAST        0x8                // srcu_read_lock_fast().
#define SRCU_READ_FLAVOR_ALL   (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \
                                SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST) // All of the above.
#define SRCU_READ_FLAVOR_SLOWGP        (SRCU_READ_FLAVOR_LITE | SRCU_READ_FLAVOR_FAST)
                                                // Flavors requiring synchronize_rcu()
                                                // instead of smp_mb().
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);

#ifdef CONFIG_TINY_SRCU
#include <linux/srcutiny.h>
#elif defined(CONFIG_TREE_SRCU)
#include <linux/srcutree.h>
#else
#error "Unknown SRCU implementation specified to kernel configuration"
#endif

void call_srcu(struct srcu_struct *ssp, struct rcu_head *head,
                void (*func)(struct rcu_head *head));
void cleanup_srcu_struct(struct srcu_struct *ssp);
void synchronize_srcu(struct srcu_struct *ssp);

#define SRCU_GET_STATE_COMPLETED 0x1

/**
 * get_completed_synchronize_srcu - Return a pre-completed polled state cookie
 *
 * Returns a value that poll_state_synchronize_srcu() will always treat
 * as a cookie whose grace period has already completed.
 */
static inline unsigned long get_completed_synchronize_srcu(void)
{
        return SRCU_GET_STATE_COMPLETED;
}

unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);

// Maximum number of unsigned long values corresponding to
// not-yet-completed SRCU grace periods.
#define NUM_ACTIVE_SRCU_POLL_OLDSTATE 2

/**
 * same_state_synchronize_srcu - Are two old-state values identical?
 * @oldstate1: First old-state value.
 * @oldstate2: Second old-state value.
 *
 * The two old-state values must have been obtained from either
 * get_state_synchronize_srcu(), start_poll_synchronize_srcu(), or
 * get_completed_synchronize_srcu().  Returns @true if the two values are
 * identical and @false otherwise.  This allows structures whose lifetimes
 * are tracked by old-state values to push these values to a list header,
 * allowing those structures to be slightly smaller.
 */
static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned long oldstate2)
{
        return oldstate1 == oldstate2;
}

#ifdef CONFIG_NEED_SRCU_NMI_SAFE
int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp);
void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp);
#else
static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)
{
        return __srcu_read_lock(ssp);
}
static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
{
        __srcu_read_unlock(ssp, idx);
}
#endif /* CONFIG_NEED_SRCU_NMI_SAFE */

void srcu_init(void);

#ifdef CONFIG_DEBUG_LOCK_ALLOC

/**
 * srcu_read_lock_held - might we be in SRCU read-side critical section?
 * @ssp: The srcu_struct structure to check
 *
 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
 * read-side critical section.  In absence of CONFIG_DEBUG_LOCK_ALLOC,
 * this assumes we are in an SRCU read-side critical section unless it can
 * prove otherwise.
 *
 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
 * and while lockdep is disabled.
 *
 * Note that SRCU is based on its own statemachine and it doesn't
 * relies on normal RCU, it can be called from the CPU which
 * is in the idle loop from an RCU point of view or offline.
 */
static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
        if (!debug_lockdep_rcu_enabled())
                return 1;
        return lock_is_held(&ssp->dep_map);
}

/*
 * Annotations provide deadlock detection for SRCU.
 *
 * Similar to other lockdep annotations, except there is an additional
 * srcu_lock_sync(), which is basically an empty *write*-side critical section,
 * see lock_sync() for more information.
 */

/* Annotates a srcu_read_lock() */
static inline void srcu_lock_acquire(struct lockdep_map *map)
{
        lock_map_acquire_read(map);
}

/* Annotates a srcu_read_lock() */
static inline void srcu_lock_release(struct lockdep_map *map)
{
        lock_map_release(map);
}

/* Annotates a synchronize_srcu() */
static inline void srcu_lock_sync(struct lockdep_map *map)
{
        lock_map_sync(map);
}

#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

static inline int srcu_read_lock_held(const struct srcu_struct *ssp)
{
        return 1;
}

#define srcu_lock_acquire(m) do { } while (0)
#define srcu_lock_release(m) do { } while (0)
#define srcu_lock_sync(m) do { } while (0)

#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */


/**
 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 * @c: condition to check for update-side use
 *
 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
 * critical section will result in an RCU-lockdep splat, unless @c evaluates
 * to 1.  The @c argument will normally be a logical expression containing
 * lockdep_is_held() calls.
 */
#define srcu_dereference_check(p, ssp, c) \
        __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
                                (c) || srcu_read_lock_held(ssp), __rcu)

/**
 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 *
 * Makes rcu_dereference_check() do the dirty work.  If PROVE_RCU
 * is enabled, invoking this outside of an RCU read-side critical
 * section will result in an RCU-lockdep splat.
 */
#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0)

/**
 * srcu_dereference_notrace - no tracing and no lockdep calls from here
 * @p: the pointer to fetch and protect for later dereferencing
 * @ssp: pointer to the srcu_struct, which is used to check that we
 *        really are in an SRCU read-side critical section.
 */
#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1)

/**
 * srcu_read_lock - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section.  Note that SRCU read-side
 * critical sections may be nested.  However, it is illegal to
 * call anything that waits on an SRCU grace period for the same
 * srcu_struct, whether directly or indirectly.  Please note that
 * one way to indirectly wait on an SRCU grace period is to acquire
 * a mutex that is held elsewhere while calling synchronize_srcu() or
 * synchronize_srcu_expedited().
 *
 * The return value from srcu_read_lock() is guaranteed to be
 * non-negative.  This value must be passed unaltered to the matching
 * srcu_read_unlock().  Note that srcu_read_lock() and the matching
 * srcu_read_unlock() must occur in the same context, for example, it is
 * illegal to invoke srcu_read_unlock() in an irq handler if the matching
 * srcu_read_lock() was invoked in process context.  Or, for that matter to
 * invoke srcu_read_unlock() from one task and the matching srcu_read_lock()
 * from another.
 */
static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        retval = __srcu_read_lock(ssp);
        srcu_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_read_lock_fast - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but for a light-weight
 * smp_mb()-free reader.  See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_fast() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.  Note that grace-period auto-expediting is disabled for _fast
 * srcu_struct structures because auto-expedited grace periods invoke
 * synchronize_rcu_expedited(), IPIs and all.
 *
 * Note that srcu_read_lock_fast() can be invoked only from those contexts
 * where RCU is watching, that is, from contexts where it would be legal
 * to invoke rcu_read_lock().  Otherwise, lockdep will complain.
 */
static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp)
{
        struct srcu_ctr __percpu *retval;

        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
        retval = __srcu_read_lock_fast(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_down_read_fast - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter a semaphore-like SRCU read-side critical section, but for
 * a light-weight smp_mb()-free reader.  See srcu_read_lock_fast() and
 * srcu_down_read() for more information.
 *
 * The same srcu_struct may be used concurrently by srcu_down_read_fast()
 * and srcu_read_lock_fast().
 */
static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp)
{
        WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_FAST);
        return __srcu_read_lock_fast(ssp);
}

/**
 * srcu_read_lock_lite - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but for a light-weight
 * smp_mb()-free reader.  See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_lite() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.  Note that grace-period auto-expediting is disabled for _lite
 * srcu_struct structures because auto-expedited grace periods invoke
 * synchronize_rcu_expedited(), IPIs and all.
 *
 * Note that srcu_read_lock_lite() can be invoked only from those contexts
 * where RCU is watching, that is, from contexts where it would be legal
 * to invoke rcu_read_lock().  Otherwise, lockdep will complain.
 */
static inline int srcu_read_lock_lite(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor_force(ssp, SRCU_READ_FLAVOR_LITE);
        retval = __srcu_read_lock_lite(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/**
 * srcu_read_lock_nmisafe - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter an SRCU read-side critical section, but in an NMI-safe manner.
 * See srcu_read_lock() for more information.
 *
 * If srcu_read_lock_nmisafe() is ever used on an srcu_struct structure,
 * then none of the other flavors may be used, whether before, during,
 * or after.
 */
static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
        retval = __srcu_read_lock_nmisafe(ssp);
        rcu_try_lock_acquire(&ssp->dep_map);
        return retval;
}

/* Used by tracing, cannot be traced and cannot invoke lockdep. */
static inline notrace int
srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp)
{
        int retval;

        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        retval = __srcu_read_lock(ssp);
        return retval;
}

/**
 * srcu_down_read - register a new reader for an SRCU-protected structure.
 * @ssp: srcu_struct in which to register the new reader.
 *
 * Enter a semaphore-like SRCU read-side critical section.  Note that
 * SRCU read-side critical sections may be nested.  However, it is
 * illegal to call anything that waits on an SRCU grace period for the
 * same srcu_struct, whether directly or indirectly.  Please note that
 * one way to indirectly wait on an SRCU grace period is to acquire
 * a mutex that is held elsewhere while calling synchronize_srcu() or
 * synchronize_srcu_expedited().  But if you want lockdep to help you
 * keep this stuff straight, you should instead use srcu_read_lock().
 *
 * The semaphore-like nature of srcu_down_read() means that the matching
 * srcu_up_read() can be invoked from some other context, for example,
 * from some other task or from an irq handler.  However, neither
 * srcu_down_read() nor srcu_up_read() may be invoked from an NMI handler.
 *
 * Calls to srcu_down_read() may be nested, similar to the manner in
 * which calls to down_read() may be nested.  The same srcu_struct may be
 * used concurrently by srcu_down_read() and srcu_read_lock().
 */
static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp)
{
        WARN_ON_ONCE(in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        return __srcu_read_lock(ssp);
}

/**
 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock().
 *
 * Exit an SRCU read-side critical section.
 */
static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock(ssp, idx);
}

/**
 * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @scp: return value from corresponding srcu_read_lock_fast().
 *
 * Exit a light-weight SRCU read-side critical section.
 */
static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
        __releases(ssp)
{
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_fast(ssp, scp);
}

/**
 * srcu_up_read_fast - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @scp: return value from corresponding srcu_read_lock_fast().
 *
 * Exit an SRCU read-side critical section, but not necessarily from
 * the same context as the maching srcu_down_read_fast().
 */
static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp)
        __releases(ssp)
{
        WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST);
        __srcu_read_unlock_fast(ssp, scp);
}

/**
 * srcu_read_unlock_lite - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock_lite().
 *
 * Exit a light-weight SRCU read-side critical section.
 */
static inline void srcu_read_unlock_lite(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_LITE);
        srcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_lite(ssp, idx);
}

/**
 * srcu_read_unlock_nmisafe - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock_nmisafe().
 *
 * Exit an SRCU read-side critical section, but in an NMI-safe manner.
 */
static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI);
        rcu_lock_release(&ssp->dep_map);
        __srcu_read_unlock_nmisafe(ssp, idx);
}

/* Used by tracing, cannot be traced and cannot call lockdep. */
static inline notrace void
srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp)
{
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        __srcu_read_unlock(ssp, idx);
}

/**
 * srcu_up_read - unregister a old reader from an SRCU-protected structure.
 * @ssp: srcu_struct in which to unregister the old reader.
 * @idx: return value from corresponding srcu_read_lock().
 *
 * Exit an SRCU read-side critical section, but not necessarily from
 * the same context as the maching srcu_down_read().
 */
static inline void srcu_up_read(struct srcu_struct *ssp, int idx)
        __releases(ssp)
{
        WARN_ON_ONCE(idx & ~0x1);
        WARN_ON_ONCE(in_nmi());
        srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL);
        __srcu_read_unlock(ssp, idx);
}

/**
 * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock
 *
 * Converts the preceding srcu_read_unlock into a two-way memory barrier.
 *
 * Call this after srcu_read_unlock, to guarantee that all memory operations
 * that occur after smp_mb__after_srcu_read_unlock will appear to happen after
 * the preceding srcu_read_unlock.
 */
static inline void smp_mb__after_srcu_read_unlock(void)
{
        /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */
}

/**
 * smp_mb__after_srcu_read_lock - ensure full ordering after srcu_read_lock
 *
 * Converts the preceding srcu_read_lock into a two-way memory barrier.
 *
 * Call this after srcu_read_lock, to guarantee that all memory operations
 * that occur after smp_mb__after_srcu_read_lock will appear to happen after
 * the preceding srcu_read_lock.
 */
static inline void smp_mb__after_srcu_read_lock(void)
{
        /* __srcu_read_lock has smp_mb() internally so nothing to do here. */
}

DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct,
                    _T->idx = srcu_read_lock(_T->lock),
                    srcu_read_unlock(_T->lock, _T->idx),
                    int idx)

#endif










 6151 
















































 5085 




 5085 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM csd

#if !defined(_TRACE_CSD_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_CSD_H

#include <linux/tracepoint.h>

TRACE_EVENT(csd_queue_cpu,

        TP_PROTO(const unsigned int cpu,
                unsigned long callsite,
                smp_call_func_t func,
                call_single_data_t *csd),

        TP_ARGS(cpu, callsite, func, csd),

        TP_STRUCT__entry(
                __field(unsigned int, cpu)
                __field(void *, callsite)
                __field(void *, func)
                __field(void *, csd)
                ),

            TP_fast_assign(
                __entry->cpu = cpu;
                __entry->callsite = (void *)callsite;
                __entry->func = func;
                __entry->csd  = csd;
                ),

        TP_printk("cpu=%u callsite=%pS func=%ps csd=%p",
                __entry->cpu, __entry->callsite, __entry->func, __entry->csd)
        );

/*
 * Tracepoints for a function which is called as an effect of smp_call_function.*
 */
DECLARE_EVENT_CLASS(csd_function,

        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),

        TP_ARGS(func, csd),

        TP_STRUCT__entry(
                __field(void *,        func)
                __field(void *,        csd)
        ),

        TP_fast_assign(
                __entry->func        = func;
                __entry->csd        = csd;
        ),

        TP_printk("func=%ps, csd=%p", __entry->func, __entry->csd)
);

DEFINE_EVENT(csd_function, csd_function_entry,
        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
        TP_ARGS(func, csd)
);

DEFINE_EVENT(csd_function, csd_function_exit,
        TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
        TP_ARGS(func, csd)
);

#endif /* _TRACE_CSD_H */

/* This part must be outside protection */
#include <trace/define_trace.h>










































    6 







    6 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Copyright (C) 2019 Oracle.  All Rights Reserved.
 * Author: Darrick J. Wong <darrick.wong@oracle.com>
 */
#ifndef __XFS_PWORK_H__
#define __XFS_PWORK_H__

struct xfs_pwork;
struct xfs_mount;

typedef int (*xfs_pwork_work_fn)(struct xfs_mount *mp, struct xfs_pwork *pwork);

/*
 * Parallel work coordination structure.
 */
struct xfs_pwork_ctl {
        struct workqueue_struct        *wq;
        struct xfs_mount        *mp;
        xfs_pwork_work_fn        work_fn;
        struct wait_queue_head        poll_wait;
        atomic_t                nr_work;
        int                        error;
};

/*
 * Embed this parallel work control item inside your own work structure,
 * then queue work with it.
 */
struct xfs_pwork {
        struct work_struct        work;
        struct xfs_pwork_ctl        *pctl;
};

#define XFS_PWORK_SINGLE_THREADED        { .pctl = NULL }

/* Have we been told to abort? */
static inline bool
xfs_pwork_ctl_want_abort(
        struct xfs_pwork_ctl        *pctl)
{
        return pctl && pctl->error;
}

/* Have we been told to abort? */
static inline bool
xfs_pwork_want_abort(
        struct xfs_pwork        *pwork)
{
        return xfs_pwork_ctl_want_abort(pwork->pctl);
}

int xfs_pwork_init(struct xfs_mount *mp, struct xfs_pwork_ctl *pctl,
                xfs_pwork_work_fn work_fn, const char *tag);
void xfs_pwork_queue(struct xfs_pwork_ctl *pctl, struct xfs_pwork *pwork);
int xfs_pwork_destroy(struct xfs_pwork_ctl *pctl);
void xfs_pwork_poll(struct xfs_pwork_ctl *pctl);

#endif /* __XFS_PWORK_H__ */



























    7 




    2 





    6 







    5 










    6 



















   20 


   20 




   19 










   17 


    2 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
// SPDX-License-Identifier: GPL-2.0-only

#include "netlink.h"
#include "common.h"
#include "bitset.h"

struct debug_req_info {
        struct ethnl_req_info                base;
};

struct debug_reply_data {
        struct ethnl_reply_data                base;
        u32                                msg_mask;
};

#define DEBUG_REPDATA(__reply_base) \
        container_of(__reply_base, struct debug_reply_data, base)

const struct nla_policy ethnl_debug_get_policy[] = {
        [ETHTOOL_A_DEBUG_HEADER]        =
                NLA_POLICY_NESTED(ethnl_header_policy),
};

static int debug_prepare_data(const struct ethnl_req_info *req_base,
                              struct ethnl_reply_data *reply_base,
                              const struct genl_info *info)
{
        struct debug_reply_data *data = DEBUG_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        int ret;

        if (!dev->ethtool_ops->get_msglevel)
                return -EOPNOTSUPP;

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;
        data->msg_mask = dev->ethtool_ops->get_msglevel(dev);
        ethnl_ops_complete(dev);

        return 0;
}

static int debug_reply_size(const struct ethnl_req_info *req_base,
                            const struct ethnl_reply_data *reply_base)
{
        const struct debug_reply_data *data = DEBUG_REPDATA(reply_base);
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;

        return ethnl_bitset32_size(&data->msg_mask, NULL, NETIF_MSG_CLASS_COUNT,
                                   netif_msg_class_names, compact);
}

static int debug_fill_reply(struct sk_buff *skb,
                            const struct ethnl_req_info *req_base,
                            const struct ethnl_reply_data *reply_base)
{
        const struct debug_reply_data *data = DEBUG_REPDATA(reply_base);
        bool compact = req_base->flags & ETHTOOL_FLAG_COMPACT_BITSETS;

        return ethnl_put_bitset32(skb, ETHTOOL_A_DEBUG_MSGMASK, &data->msg_mask,
                                  NULL, NETIF_MSG_CLASS_COUNT,
                                  netif_msg_class_names, compact);
}

/* DEBUG_SET */

const struct nla_policy ethnl_debug_set_policy[] = {
        [ETHTOOL_A_DEBUG_HEADER]        =
                NLA_POLICY_NESTED(ethnl_header_policy),
        [ETHTOOL_A_DEBUG_MSGMASK]        = { .type = NLA_NESTED },
};

static int
ethnl_set_debug_validate(struct ethnl_req_info *req_info,
                         struct genl_info *info)
{
        const struct ethtool_ops *ops = req_info->dev->ethtool_ops;

        return ops->get_msglevel && ops->set_msglevel ? 1 : -EOPNOTSUPP;
}

static int
ethnl_set_debug(struct ethnl_req_info *req_info, struct genl_info *info)
{
        struct net_device *dev = req_info->dev;
        struct nlattr **tb = info->attrs;
        bool mod = false;
        u32 msg_mask;
        int ret;

        msg_mask = dev->ethtool_ops->get_msglevel(dev);
        ret = ethnl_update_bitset32(&msg_mask, NETIF_MSG_CLASS_COUNT,
                                    tb[ETHTOOL_A_DEBUG_MSGMASK],
                                    netif_msg_class_names, info->extack, &mod);
        if (ret < 0 || !mod)
                return ret;

        dev->ethtool_ops->set_msglevel(dev, msg_mask);
        return 1;
}

const struct ethnl_request_ops ethnl_debug_request_ops = {
        .request_cmd                = ETHTOOL_MSG_DEBUG_GET,
        .reply_cmd                = ETHTOOL_MSG_DEBUG_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_DEBUG_HEADER,
        .req_info_size                = sizeof(struct debug_req_info),
        .reply_data_size        = sizeof(struct debug_reply_data),

        .prepare_data                = debug_prepare_data,
        .reply_size                = debug_reply_size,
        .fill_reply                = debug_fill_reply,

        .set_validate                = ethnl_set_debug_validate,
        .set                        = ethnl_set_debug,
        .set_ntf_cmd                = ETHTOOL_MSG_DEBUG_NTF,
};





















   15 














    8 
   15 
   15 











    8 




    5 



   15 





    6 



    2 

    3 


    5 






















    6 








    2 

    5 

    3 


    6 







    8 







    8 
    8 
    8 
    6 
    6 


    2 














    2 




    6 






    6 
    6 
    6 
    6 


















    3 









    3 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
// SPDX-License-Identifier: GPL-2.0-only
/*
 * dma-fence-util: misc functions for dma_fence objects
 *
 * Copyright (C) 2022 Advanced Micro Devices, Inc.
 * Authors:
 *        Christian König <christian.koenig@amd.com>
 */

#include <linux/dma-fence.h>
#include <linux/dma-fence-array.h>
#include <linux/dma-fence-chain.h>
#include <linux/dma-fence-unwrap.h>
#include <linux/slab.h>
#include <linux/sort.h>

/* Internal helper to start new array iteration, don't use directly */
static struct dma_fence *
__dma_fence_unwrap_array(struct dma_fence_unwrap *cursor)
{
        cursor->array = dma_fence_chain_contained(cursor->chain);
        cursor->index = 0;
        return dma_fence_array_first(cursor->array);
}

/**
 * dma_fence_unwrap_first - return the first fence from fence containers
 * @head: the entrypoint into the containers
 * @cursor: current position inside the containers
 *
 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the
 * first fence.
 */
struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head,
                                         struct dma_fence_unwrap *cursor)
{
        cursor->chain = dma_fence_get(head);
        return __dma_fence_unwrap_array(cursor);
}
EXPORT_SYMBOL_GPL(dma_fence_unwrap_first);

/**
 * dma_fence_unwrap_next - return the next fence from a fence containers
 * @cursor: current position inside the containers
 *
 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return
 * the next fence from them.
 */
struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor)
{
        struct dma_fence *tmp;

        ++cursor->index;
        tmp = dma_fence_array_next(cursor->array, cursor->index);
        if (tmp)
                return tmp;

        cursor->chain = dma_fence_chain_walk(cursor->chain);
        return __dma_fence_unwrap_array(cursor);
}
EXPORT_SYMBOL_GPL(dma_fence_unwrap_next);


static int fence_cmp(const void *_a, const void *_b)
{
        struct dma_fence *a = *(struct dma_fence **)_a;
        struct dma_fence *b = *(struct dma_fence **)_b;

        if (a->context < b->context)
                return -1;
        else if (a->context > b->context)
                return 1;

        if (dma_fence_is_later(b, a))
                return 1;
        else if (dma_fence_is_later(a, b))
                return -1;

        return 0;
}

/**
 * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers
 * @fences:     Array of dma_fence pointers to be deduplicated
 * @num_fences: Number of entries in the @fences array
 *
 * Sorts the input array by context, then removes duplicate
 * fences with the same context, keeping only the most recent one.
 *
 * The array is modified in-place and unreferenced duplicate fences are released
 * via dma_fence_put(). The function returns the new number of fences after
 * deduplication.
 *
 * Return: Number of unique fences remaining in the array.
 */
int dma_fence_dedup_array(struct dma_fence **fences, int num_fences)
{
        int i, j;

        sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL);

        /*
         * Only keep the most recent fence for each context.
         */
        j = 0;
        for (i = 1; i < num_fences; i++) {
                if (fences[i]->context == fences[j]->context)
                        dma_fence_put(fences[i]);
                else
                        fences[++j] = fences[i];
        }

        return ++j;
}
EXPORT_SYMBOL_GPL(dma_fence_dedup_array);

/* Implementation for the dma_fence_merge() marco, don't use directly */
struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
                                           struct dma_fence **fences,
                                           struct dma_fence_unwrap *iter)
{
        struct dma_fence *tmp, *unsignaled = NULL, **array;
        struct dma_fence_array *result;
        ktime_t timestamp;
        int i, count;

        count = 0;
        timestamp = ns_to_ktime(0);
        for (i = 0; i < num_fences; ++i) {
                dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
                        if (!dma_fence_is_signaled(tmp)) {
                                dma_fence_put(unsignaled);
                                unsignaled = dma_fence_get(tmp);
                                ++count;
                        } else {
                                ktime_t t = dma_fence_timestamp(tmp);

                                if (ktime_after(t, timestamp))
                                        timestamp = t;
                        }
                }
        }

        /*
         * If we couldn't find a pending fence just return a private signaled
         * fence with the timestamp of the last signaled one.
         *
         * Or if there was a single unsignaled fence left we can return it
         * directly and early since that is a major path on many workloads.
         */
        if (count == 0)
                return dma_fence_allocate_private_stub(timestamp);
        else if (count == 1)
                return unsignaled;

        dma_fence_put(unsignaled);

        array = kmalloc_array(count, sizeof(*array), GFP_KERNEL);
        if (!array)
                return NULL;

        count = 0;
        for (i = 0; i < num_fences; ++i) {
                dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) {
                        if (!dma_fence_is_signaled(tmp)) {
                                array[count++] = dma_fence_get(tmp);
                        } else {
                                ktime_t t = dma_fence_timestamp(tmp);

                                if (ktime_after(t, timestamp))
                                        timestamp = t;
                        }
                }
        }

        if (count == 0 || count == 1)
                goto return_fastpath;

        count = dma_fence_dedup_array(array, count);

        if (count > 1) {
                result = dma_fence_array_create(count, array,
                                                dma_fence_context_alloc(1),
                                                1, false);
                if (!result) {
                        for (i = 0; i < count; i++)
                                dma_fence_put(array[i]);
                        tmp = NULL;
                        goto return_tmp;
                }
                return &result->base;
        }

return_fastpath:
        if (count == 0)
                tmp = dma_fence_allocate_private_stub(timestamp);
        else
                tmp = array[0];

return_tmp:
        kfree(array);
        return tmp;
}
EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge);







































    2 
















    2 















    2 













































































































































































    1 




























    1 






    1 



































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
// SPDX-License-Identifier: GPL-2.0-only
//
// ethtool interface for Ethernet PSE (Power Sourcing Equipment)
// and PD (Powered Device)
//
// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
//

#include "common.h"
#include "linux/pse-pd/pse.h"
#include "netlink.h"
#include <linux/ethtool_netlink.h>
#include <linux/ethtool.h>
#include <linux/phy.h>

struct pse_req_info {
        struct ethnl_req_info base;
};

struct pse_reply_data {
        struct ethnl_reply_data        base;
        struct ethtool_pse_control_status status;
};

#define PSE_REPDATA(__reply_base) \
        container_of(__reply_base, struct pse_reply_data, base)

/* PSE_GET */

const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1] = {
        [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy_phy),
};

static int pse_get_pse_attributes(struct phy_device *phydev,
                                  struct netlink_ext_ack *extack,
                                  struct pse_reply_data *data)
{
        if (!phydev) {
                NL_SET_ERR_MSG(extack, "No PHY found");
                return -EOPNOTSUPP;
        }

        if (!phydev->psec) {
                NL_SET_ERR_MSG(extack, "No PSE is attached");
                return -EOPNOTSUPP;
        }

        memset(&data->status, 0, sizeof(data->status));

        return pse_ethtool_get_status(phydev->psec, extack, &data->status);
}

static int pse_prepare_data(const struct ethnl_req_info *req_base,
                            struct ethnl_reply_data *reply_base,
                            const struct genl_info *info)
{
        struct pse_reply_data *data = PSE_REPDATA(reply_base);
        struct net_device *dev = reply_base->dev;
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        ret = ethnl_ops_begin(dev);
        if (ret < 0)
                return ret;

        phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_PSE_HEADER,
                                      info->extack);
        if (IS_ERR(phydev))
                return -ENODEV;

        ret = pse_get_pse_attributes(phydev, info->extack, data);

        ethnl_ops_complete(dev);

        return ret;
}

static int pse_reply_size(const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);
        const struct ethtool_pse_control_status *st = &data->status;
        int len = 0;

        if (st->pw_d_id)
                len += nla_total_size(sizeof(u32)); /* _PSE_PW_D_ID */
        if (st->podl_admin_state > 0)
                len += nla_total_size(sizeof(u32)); /* _PODL_PSE_ADMIN_STATE */
        if (st->podl_pw_status > 0)
                len += nla_total_size(sizeof(u32)); /* _PODL_PSE_PW_D_STATUS */
        if (st->c33_admin_state > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_ADMIN_STATE */
        if (st->c33_pw_status > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_PW_D_STATUS */
        if (st->c33_pw_class > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_PW_CLASS */
        if (st->c33_actual_pw > 0)
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_ACTUAL_PW */
        if (st->c33_ext_state_info.c33_pse_ext_state > 0) {
                len += nla_total_size(sizeof(u32)); /* _C33_PSE_EXT_STATE */
                if (st->c33_ext_state_info.__c33_pse_ext_substate > 0)
                        /* _C33_PSE_EXT_SUBSTATE */
                        len += nla_total_size(sizeof(u32));
        }
        if (st->c33_avail_pw_limit > 0)
                /* _C33_AVAIL_PSE_PW_LIMIT */
                len += nla_total_size(sizeof(u32));
        if (st->c33_pw_limit_nb_ranges > 0)
                /* _C33_PSE_PW_LIMIT_RANGES */
                len += st->c33_pw_limit_nb_ranges *
                       (nla_total_size(0) +
                        nla_total_size(sizeof(u32)) * 2);
        if (st->prio_max)
                /* _PSE_PRIO_MAX + _PSE_PRIO */
                len += nla_total_size(sizeof(u32)) * 2;

        return len;
}

static int pse_put_pw_limit_ranges(struct sk_buff *skb,
                                   const struct ethtool_pse_control_status *st)
{
        const struct ethtool_c33_pse_pw_limit_range *pw_limit_ranges;
        int i;

        pw_limit_ranges = st->c33_pw_limit_ranges;
        for (i = 0; i < st->c33_pw_limit_nb_ranges; i++) {
                struct nlattr *nest;

                nest = nla_nest_start(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_RANGES);
                if (!nest)
                        return -EMSGSIZE;

                if (nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_MIN,
                                pw_limit_ranges->min) ||
                    nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_LIMIT_MAX,
                                pw_limit_ranges->max)) {
                        nla_nest_cancel(skb, nest);
                        return -EMSGSIZE;
                }
                nla_nest_end(skb, nest);
                pw_limit_ranges++;
        }

        return 0;
}

static int pse_fill_reply(struct sk_buff *skb,
                          const struct ethnl_req_info *req_base,
                          const struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);
        const struct ethtool_pse_control_status *st = &data->status;

        if (st->pw_d_id &&
            nla_put_u32(skb, ETHTOOL_A_PSE_PW_D_ID,
                        st->pw_d_id))
                return -EMSGSIZE;

        if (st->podl_admin_state > 0 &&
            nla_put_u32(skb, ETHTOOL_A_PODL_PSE_ADMIN_STATE,
                        st->podl_admin_state))
                return -EMSGSIZE;

        if (st->podl_pw_status > 0 &&
            nla_put_u32(skb, ETHTOOL_A_PODL_PSE_PW_D_STATUS,
                        st->podl_pw_status))
                return -EMSGSIZE;

        if (st->c33_admin_state > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_ADMIN_STATE,
                        st->c33_admin_state))
                return -EMSGSIZE;

        if (st->c33_pw_status > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_D_STATUS,
                        st->c33_pw_status))
                return -EMSGSIZE;

        if (st->c33_pw_class > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_PW_CLASS,
                        st->c33_pw_class))
                return -EMSGSIZE;

        if (st->c33_actual_pw > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_ACTUAL_PW,
                        st->c33_actual_pw))
                return -EMSGSIZE;

        if (st->c33_ext_state_info.c33_pse_ext_state > 0) {
                if (nla_put_u32(skb, ETHTOOL_A_C33_PSE_EXT_STATE,
                                st->c33_ext_state_info.c33_pse_ext_state))
                        return -EMSGSIZE;

                if (st->c33_ext_state_info.__c33_pse_ext_substate > 0 &&
                    nla_put_u32(skb, ETHTOOL_A_C33_PSE_EXT_SUBSTATE,
                                st->c33_ext_state_info.__c33_pse_ext_substate))
                        return -EMSGSIZE;
        }

        if (st->c33_avail_pw_limit > 0 &&
            nla_put_u32(skb, ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT,
                        st->c33_avail_pw_limit))
                return -EMSGSIZE;

        if (st->c33_pw_limit_nb_ranges > 0 &&
            pse_put_pw_limit_ranges(skb, st))
                return -EMSGSIZE;

        if (st->prio_max &&
            (nla_put_u32(skb, ETHTOOL_A_PSE_PRIO_MAX, st->prio_max) ||
             nla_put_u32(skb, ETHTOOL_A_PSE_PRIO, st->prio)))
                return -EMSGSIZE;

        return 0;
}

static void pse_cleanup_data(struct ethnl_reply_data *reply_base)
{
        const struct pse_reply_data *data = PSE_REPDATA(reply_base);

        kfree(data->status.c33_pw_limit_ranges);
}

/* PSE_SET */

const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1] = {
        [ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy_phy),
        [ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] =
                NLA_POLICY_RANGE(NLA_U32, ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED,
                                 ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED),
        [ETHTOOL_A_C33_PSE_ADMIN_CONTROL] =
                NLA_POLICY_RANGE(NLA_U32, ETHTOOL_C33_PSE_ADMIN_STATE_DISABLED,
                                 ETHTOOL_C33_PSE_ADMIN_STATE_ENABLED),
        [ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT] = { .type = NLA_U32 },
        [ETHTOOL_A_PSE_PRIO] = { .type = NLA_U32 },
};

static int
ethnl_set_pse_validate(struct phy_device *phydev, struct genl_info *info)
{
        struct nlattr **tb = info->attrs;

        if (IS_ERR_OR_NULL(phydev)) {
                NL_SET_ERR_MSG(info->extack, "No PHY is attached");
                return -EOPNOTSUPP;
        }

        if (!phydev->psec) {
                NL_SET_ERR_MSG(info->extack, "No PSE is attached");
                return -EOPNOTSUPP;
        }

        if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] &&
            !pse_has_podl(phydev->psec)) {
                NL_SET_ERR_MSG_ATTR(info->extack,
                                    tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL],
                                    "setting PoDL PSE admin control not supported");
                return -EOPNOTSUPP;
        }
        if (tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL] &&
            !pse_has_c33(phydev->psec)) {
                NL_SET_ERR_MSG_ATTR(info->extack,
                                    tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL],
                                    "setting C33 PSE admin control not supported");
                return -EOPNOTSUPP;
        }

        return 0;
}

static int
ethnl_set_pse(struct ethnl_req_info *req_info, struct genl_info *info)
{
        struct nlattr **tb = info->attrs;
        struct phy_device *phydev;
        int ret;

        phydev = ethnl_req_get_phydev(req_info, tb, ETHTOOL_A_PSE_HEADER,
                                      info->extack);
        ret = ethnl_set_pse_validate(phydev, info);
        if (ret)
                return ret;

        if (tb[ETHTOOL_A_PSE_PRIO]) {
                unsigned int prio;

                prio = nla_get_u32(tb[ETHTOOL_A_PSE_PRIO]);
                ret = pse_ethtool_set_prio(phydev->psec, info->extack, prio);
                if (ret)
                        return ret;
        }

        if (tb[ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT]) {
                unsigned int pw_limit;

                pw_limit = nla_get_u32(tb[ETHTOOL_A_C33_PSE_AVAIL_PW_LIMIT]);
                ret = pse_ethtool_set_pw_limit(phydev->psec, info->extack,
                                               pw_limit);
                if (ret)
                        return ret;
        }

        /* These values are already validated by the ethnl_pse_set_policy */
        if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] ||
            tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL]) {
                struct pse_control_config config = {};

                if (tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL])
                        config.podl_admin_control = nla_get_u32(tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]);
                if (tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL])
                        config.c33_admin_control = nla_get_u32(tb[ETHTOOL_A_C33_PSE_ADMIN_CONTROL]);

                /* pse_ethtool_set_config() will do nothing if the config
                 * is zero
                 */
                ret = pse_ethtool_set_config(phydev->psec, info->extack,
                                             &config);
                if (ret)
                        return ret;
        }

        /* Return errno or zero - PSE has no notification */
        return ret;
}

const struct ethnl_request_ops ethnl_pse_request_ops = {
        .request_cmd                = ETHTOOL_MSG_PSE_GET,
        .reply_cmd                = ETHTOOL_MSG_PSE_GET_REPLY,
        .hdr_attr                = ETHTOOL_A_PSE_HEADER,
        .req_info_size                = sizeof(struct pse_req_info),
        .reply_data_size        = sizeof(struct pse_reply_data),

        .prepare_data                = pse_prepare_data,
        .reply_size                = pse_reply_size,
        .fill_reply                = pse_fill_reply,
        .cleanup_data                = pse_cleanup_data,

        .set                        = ethnl_set_pse,
        /* PSE has no notification */
};

void ethnl_pse_send_ntf(struct net_device *netdev, unsigned long notifs)
{
        void *reply_payload;
        struct sk_buff *skb;
        int reply_len;
        int ret;

        ASSERT_RTNL();

        if (!netdev || !notifs)
                return;

        reply_len = ethnl_reply_header_size() +
                    nla_total_size(sizeof(u32)); /* _PSE_NTF_EVENTS */

        skb = genlmsg_new(reply_len, GFP_KERNEL);
        if (!skb)
                return;

        reply_payload = ethnl_bcastmsg_put(skb, ETHTOOL_MSG_PSE_NTF);
        if (!reply_payload)
                goto err_skb;

        ret = ethnl_fill_reply_header(skb, netdev, ETHTOOL_A_PSE_NTF_HEADER);
        if (ret < 0)
                goto err_skb;

        if (nla_put_uint(skb, ETHTOOL_A_PSE_NTF_EVENTS, notifs))
                goto err_skb;

        genlmsg_end(skb, reply_payload);
        ethnl_multicast(skb, netdev);
        return;

err_skb:
        nlmsg_free(skb);
}
EXPORT_SYMBOL_GPL(ethnl_pse_send_ntf);



























































































































   25 











   35 

   15 











































   14 




   16 





 1815 


    2 


 1809 





   31 




   33 





















   23 




   25 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
// SPDX-License-Identifier: GPL-2.0-only
/*
 * The "user cache".
 *
 * (C) Copyright 1991-2000 Linus Torvalds
 *
 * We have a per-user structure to keep track of how many
 * processes, files etc the user has claimed, in order to be
 * able to have per-user limits for system resources. 
 */

#include <linux/init.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/bitops.h>
#include <linux/key.h>
#include <linux/sched/user.h>
#include <linux/interrupt.h>
#include <linux/export.h>
#include <linux/user_namespace.h>
#include <linux/binfmts.h>
#include <linux/proc_ns.h>

#if IS_ENABLED(CONFIG_BINFMT_MISC)
struct binfmt_misc init_binfmt_misc = {
        .entries = LIST_HEAD_INIT(init_binfmt_misc.entries),
        .enabled = true,
        .entries_lock = __RW_LOCK_UNLOCKED(init_binfmt_misc.entries_lock),
};
EXPORT_SYMBOL_GPL(init_binfmt_misc);
#endif

/*
 * userns count is 1 for root user, 1 for init_uts_ns,
 * and 1 for... ?
 */
struct user_namespace init_user_ns = {
        .uid_map = {
                {
                        .extent[0] = {
                                .first = 0,
                                .lower_first = 0,
                                .count = 4294967295U,
                        },
                        .nr_extents = 1,
                },
        },
        .gid_map = {
                {
                        .extent[0] = {
                                .first = 0,
                                .lower_first = 0,
                                .count = 4294967295U,
                        },
                        .nr_extents = 1,
                },
        },
        .projid_map = {
                {
                        .extent[0] = {
                                .first = 0,
                                .lower_first = 0,
                                .count = 4294967295U,
                        },
                        .nr_extents = 1,
                },
        },
        .ns.count = REFCOUNT_INIT(3),
        .owner = GLOBAL_ROOT_UID,
        .group = GLOBAL_ROOT_GID,
        .ns.inum = PROC_USER_INIT_INO,
#ifdef CONFIG_USER_NS
        .ns.ops = &userns_operations,
#endif
        .flags = USERNS_INIT_FLAGS,
#ifdef CONFIG_KEYS
        .keyring_name_list = LIST_HEAD_INIT(init_user_ns.keyring_name_list),
        .keyring_sem = __RWSEM_INITIALIZER(init_user_ns.keyring_sem),
#endif
#if IS_ENABLED(CONFIG_BINFMT_MISC)
        .binfmt_misc = &init_binfmt_misc,
#endif
};
EXPORT_SYMBOL_GPL(init_user_ns);

/*
 * UID task count cache, to get fast user lookup in "alloc_uid"
 * when changing user ID's (ie setuid() and friends).
 */

#define UIDHASH_BITS        (IS_ENABLED(CONFIG_BASE_SMALL) ? 3 : 7)
#define UIDHASH_SZ        (1 << UIDHASH_BITS)
#define UIDHASH_MASK                (UIDHASH_SZ - 1)
#define __uidhashfn(uid)        (((uid >> UIDHASH_BITS) + uid) & UIDHASH_MASK)
#define uidhashentry(uid)        (uidhash_table + __uidhashfn((__kuid_val(uid))))

static struct kmem_cache *uid_cachep;
static struct hlist_head uidhash_table[UIDHASH_SZ];

/*
 * The uidhash_lock is mostly taken from process context, but it is
 * occasionally also taken from softirq/tasklet context, when
 * task-structs get RCU-freed. Hence all locking must be softirq-safe.
 * But free_uid() is also called with local interrupts disabled, and running
 * local_bh_enable() with local interrupts disabled is an error - we'll run
 * softirq callbacks, and they can unconditionally enable interrupts, and
 * the caller of free_uid() didn't expect that..
 */
static DEFINE_SPINLOCK(uidhash_lock);

/* root_user.__count is 1, for init task cred */
struct user_struct root_user = {
        .__count        = REFCOUNT_INIT(1),
        .uid                = GLOBAL_ROOT_UID,
        .ratelimit        = RATELIMIT_STATE_INIT(root_user.ratelimit, 0, 0),
};

/*
 * These routines must be called with the uidhash spinlock held!
 */
static void uid_hash_insert(struct user_struct *up, struct hlist_head *hashent)
{
        hlist_add_head(&up->uidhash_node, hashent);
}

static void uid_hash_remove(struct user_struct *up)
{
        hlist_del_init(&up->uidhash_node);
}

static struct user_struct *uid_hash_find(kuid_t uid, struct hlist_head *hashent)
{
        struct user_struct *user;

        hlist_for_each_entry(user, hashent, uidhash_node) {
                if (uid_eq(user->uid, uid)) {
                        refcount_inc(&user->__count);
                        return user;
                }
        }

        return NULL;
}

static int user_epoll_alloc(struct user_struct *up)
{
#ifdef CONFIG_EPOLL
        return percpu_counter_init(&up->epoll_watches, 0, GFP_KERNEL);
#else
        return 0;
#endif
}

static void user_epoll_free(struct user_struct *up)
{
#ifdef CONFIG_EPOLL
        percpu_counter_destroy(&up->epoll_watches);
#endif
}

/* IRQs are disabled and uidhash_lock is held upon function entry.
 * IRQ state (as stored in flags) is restored and uidhash_lock released
 * upon function exit.
 */
static void free_user(struct user_struct *up, unsigned long flags)
        __releases(&uidhash_lock)
{
        uid_hash_remove(up);
        spin_unlock_irqrestore(&uidhash_lock, flags);
        user_epoll_free(up);
        kmem_cache_free(uid_cachep, up);
}

/*
 * Locate the user_struct for the passed UID.  If found, take a ref on it.  The
 * caller must undo that ref with free_uid().
 *
 * If the user_struct could not be found, return NULL.
 */
struct user_struct *find_user(kuid_t uid)
{
        struct user_struct *ret;
        unsigned long flags;

        spin_lock_irqsave(&uidhash_lock, flags);
        ret = uid_hash_find(uid, uidhashentry(uid));
        spin_unlock_irqrestore(&uidhash_lock, flags);
        return ret;
}

void free_uid(struct user_struct *up)
{
        unsigned long flags;

        if (!up)
                return;

        if (refcount_dec_and_lock_irqsave(&up->__count, &uidhash_lock, &flags))
                free_user(up, flags);
}
EXPORT_SYMBOL_GPL(free_uid);

struct user_struct *alloc_uid(kuid_t uid)
{
        struct hlist_head *hashent = uidhashentry(uid);
        struct user_struct *up, *new;

        spin_lock_irq(&uidhash_lock);
        up = uid_hash_find(uid, hashent);
        spin_unlock_irq(&uidhash_lock);

        if (!up) {
                new = kmem_cache_zalloc(uid_cachep, GFP_KERNEL);
                if (!new)
                        return NULL;

                new->uid = uid;
                refcount_set(&new->__count, 1);
                if (user_epoll_alloc(new)) {
                        kmem_cache_free(uid_cachep, new);
                        return NULL;
                }
                ratelimit_state_init(&new->ratelimit, HZ, 100);
                ratelimit_set_flags(&new->ratelimit, RATELIMIT_MSG_ON_RELEASE);

                /*
                 * Before adding this, check whether we raced
                 * on adding the same user already..
                 */
                spin_lock_irq(&uidhash_lock);
                up = uid_hash_find(uid, hashent);
                if (up) {
                        user_epoll_free(new);
                        kmem_cache_free(uid_cachep, new);
                } else {
                        uid_hash_insert(new, hashent);
                        up = new;
                }
                spin_unlock_irq(&uidhash_lock);
        }

        return up;
}

static int __init uid_cache_init(void)
{
        int n;

        uid_cachep = kmem_cache_create("uid_cache", sizeof(struct user_struct),
                        0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);

        for(n = 0; n < UIDHASH_SZ; ++n)
                INIT_HLIST_HEAD(uidhash_table + n);

        if (user_epoll_alloc(&root_user))
                panic("root_user epoll percpu counter alloc failed");

        /* Insert the root user immediately (init already runs as root) */
        spin_lock_irq(&uidhash_lock);
        uid_hash_insert(&root_user, uidhashentry(GLOBAL_ROOT_UID));
        spin_unlock_irq(&uidhash_lock);

        return 0;
}
subsys_initcall(uid_cache_init);







































    5 



































































  189 








   97 





























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Statically sized hash table implementation
 * (C) 2012  Sasha Levin <levinsasha928@gmail.com>
 */

#ifndef _LINUX_HASHTABLE_H
#define _LINUX_HASHTABLE_H

#include <linux/list.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/hash.h>
#include <linux/rculist.h>

#define DEFINE_HASHTABLE(name, bits)                                                \
        struct hlist_head name[1 << (bits)] =                                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DEFINE_READ_MOSTLY_HASHTABLE(name, bits)                                \
        struct hlist_head name[1 << (bits)] __read_mostly =                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DECLARE_HASHTABLE(name, bits)                                           \
        struct hlist_head name[1 << (bits)]

#define HASH_SIZE(name) (ARRAY_SIZE(name))
#define HASH_BITS(name) ilog2(HASH_SIZE(name))

/* Use hash_32 when possible to allow for fast 32bit hashing in 64bit kernels. */
#define hash_min(val, bits)                                                        \
        (sizeof(val) <= 4 ? hash_32(val, bits) : hash_long(val, bits))

static inline void __hash_init(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                INIT_HLIST_HEAD(&ht[i]);
}

/**
 * hash_init - initialize a hash table
 * @hashtable: hashtable to be initialized
 *
 * Calculates the size of the hashtable from the given parameter, otherwise
 * same as hash_init_size.
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_init(hashtable) __hash_init(hashtable, HASH_SIZE(hashtable))

/**
 * hash_add - add an object to a hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add(hashtable, node, key)                                                \
        hlist_add_head(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_add_rcu - add an object to a rcu enabled hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add_rcu(hashtable, node, key)                                        \
        hlist_add_head_rcu(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_hashed - check whether an object is in any hashtable
 * @node: the &struct hlist_node of the object to be checked
 */
static inline bool hash_hashed(struct hlist_node *node)
{
        return !hlist_unhashed(node);
}

static inline bool __hash_empty(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                if (!hlist_empty(&ht[i]))
                        return false;

        return true;
}

/**
 * hash_empty - check whether a hashtable is empty
 * @hashtable: hashtable to check
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_empty(hashtable) __hash_empty(hashtable, HASH_SIZE(hashtable))

/**
 * hash_del - remove an object from a hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del(struct hlist_node *node)
{
        hlist_del_init(node);
}

/**
 * hash_del_rcu - remove an object from a rcu enabled hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del_rcu(struct hlist_node *node)
{
        hlist_del_init_rcu(node);
}

/**
 * hash_for_each - iterate over a hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each(name, bkt, obj, member)                                \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry(obj, &name[bkt], member)

/**
 * hash_for_each_rcu - iterate over a rcu enabled hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_rcu(name, bkt, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_rcu(obj, &name[bkt], member)

/**
 * hash_for_each_safe - iterate over a hashtable safe against removal of
 * hash entry
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @tmp: a &struct hlist_node used for temporary storage
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_safe(name, bkt, tmp, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_safe(obj, tmp, &name[bkt], member)

/**
 * hash_for_each_possible - iterate over all possible objects hashing to the
 * same bucket
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible(name, obj, member, key)                        \
        hlist_for_each_entry(obj, &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_rcu - iterate over all possible objects hashing to the
 * same bucket in an rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_rcu(name, obj, member, key, cond...)        \
        hlist_for_each_entry_rcu(obj, &name[hash_min(key, HASH_BITS(name))],\
                member, ## cond)

/**
 * hash_for_each_possible_rcu_notrace - iterate over all possible objects hashing
 * to the same bucket in an rcu enabled hashtable in a rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 *
 * This is the same as hash_for_each_possible_rcu() except that it does
 * not do any RCU debugging or tracing.
 */
#define hash_for_each_possible_rcu_notrace(name, obj, member, key) \
        hlist_for_each_entry_rcu_notrace(obj, \
                &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_safe - iterate over all possible objects hashing to the
 * same bucket safe against removals
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @tmp: a &struct hlist_node used for temporary storage
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_safe(name, obj, tmp, member, key)        \
        hlist_for_each_entry_safe(obj, tmp,\
                &name[hash_min(key, HASH_BITS(name))], member)


#endif




























    8 
    8 








    8 


    8 

    8 









    8 























    8 




    8 

    8 

    8 
    8 














    8 


















    8 

    8 
    8 


    8 











    8 






    8 








    8 





    8 




















    8 
















    8 








    8 





    8 












    8 









    8 
















    8 







    8 










    8 






    8 



    8 


    8 








    8 
    8 












    8 




















    8 


    8 





    8 






















    8 







    8 



    8 
    8 

    8 
    8 

    8 








    8 









    8 
    8 























    8 



    8 
    8 






















    8 




    8 
    8 


    8 
    8 


    8 



    8 






    8 
    8 








    8 






    8 






    8 





















    8 










    8 
    8 





    8 



    8 









    8 







    8 




    8 
    8 
















    8 

    8 


    8 




    8 














    8 

































    8 

















    8 
    8 






































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
 */
#include <rdma/uverbs_ioctl.h>
#include <rdma/rdma_user_ioctl.h>
#include <linux/bitops.h>
#include "rdma_core.h"
#include "uverbs.h"

static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)
{
        return -EOPNOTSUPP;
}

static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
{
        void *elm;
        int rc;

        if (key == UVERBS_API_KEY_ERR)
                return ERR_PTR(-EOVERFLOW);

        elm = kzalloc(alloc_size, GFP_KERNEL);
        if (!elm)
                return ERR_PTR(-ENOMEM);
        rc = radix_tree_insert(&uapi->radix, key, elm);
        if (rc) {
                kfree(elm);
                return ERR_PTR(rc);
        }

        return elm;
}

static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
                              size_t alloc_size, bool *exists)
{
        void *elm;

        elm = uapi_add_elm(uapi, key, alloc_size);
        if (!IS_ERR(elm)) {
                *exists = false;
                return elm;
        }

        if (elm != ERR_PTR(-EEXIST))
                return elm;

        elm = radix_tree_lookup(&uapi->radix, key);
        if (WARN_ON(!elm))
                return ERR_PTR(-EINVAL);
        *exists = true;
        return elm;
}

static int uapi_create_write(struct uverbs_api *uapi,
                             struct ib_device *ibdev,
                             const struct uapi_definition *def,
                             u32 obj_key,
                             u32 *cur_method_key)
{
        struct uverbs_api_write_method *method_elm;
        u32 method_key = obj_key;
        bool exists;

        if (def->write.is_ex)
                method_key |= uapi_key_write_ex_method(def->write.command_num);
        else
                method_key |= uapi_key_write_method(def->write.command_num);

        method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
                                      &exists);
        if (IS_ERR(method_elm))
                return PTR_ERR(method_elm);

        if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))
                return -EINVAL;

        method_elm->is_ex = def->write.is_ex;
        method_elm->handler = def->func_write;
        if (!def->write.is_ex)
                method_elm->disabled = !(ibdev->uverbs_cmd_mask &
                                         BIT_ULL(def->write.command_num));

        if (!def->write.is_ex && def->func_write) {
                method_elm->has_udata = def->write.has_udata;
                method_elm->has_resp = def->write.has_resp;
                method_elm->req_size = def->write.req_size;
                method_elm->resp_size = def->write.resp_size;
        }

        *cur_method_key = method_key;
        return 0;
}

static int uapi_merge_method(struct uverbs_api *uapi,
                             struct uverbs_api_object *obj_elm, u32 obj_key,
                             const struct uverbs_method_def *method,
                             bool is_driver)
{
        u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
        struct uverbs_api_ioctl_method *method_elm;
        unsigned int i;
        bool exists;

        if (!method->attrs)
                return 0;

        method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
                                      &exists);
        if (IS_ERR(method_elm))
                return PTR_ERR(method_elm);
        if (exists) {
                /*
                 * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
                 */
                if (WARN_ON(method->handler))
                        return -EINVAL;
        } else {
                WARN_ON(!method->handler);
                rcu_assign_pointer(method_elm->handler, method->handler);
                if (method->handler != uverbs_destroy_def_handler)
                        method_elm->driver_method = is_driver;
        }

        for (i = 0; i != method->num_attrs; i++) {
                const struct uverbs_attr_def *attr = (*method->attrs)[i];
                struct uverbs_api_attr *attr_slot;

                if (!attr)
                        continue;

                /*
                 * ENUM_IN contains the 'ids' pointer to the driver's .rodata,
                 * so if it is specified by a driver then it always makes this
                 * into a driver method.
                 */
                if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
                        method_elm->driver_method |= is_driver;

                /*
                 * Like other uobject based things we only support a single
                 * uobject being NEW'd or DESTROY'd
                 */
                if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
                        u8 access = attr->attr.u2.objs_arr.access;

                        if (WARN_ON(access == UVERBS_ACCESS_NEW ||
                                    access == UVERBS_ACCESS_DESTROY))
                                return -EINVAL;
                }

                attr_slot =
                        uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
                                     sizeof(*attr_slot));
                /* Attributes are not allowed to be modified by drivers */
                if (IS_ERR(attr_slot))
                        return PTR_ERR(attr_slot);

                attr_slot->spec = attr->attr;
        }

        return 0;
}

static int uapi_merge_obj_tree(struct uverbs_api *uapi,
                               const struct uverbs_object_def *obj,
                               bool is_driver)
{
        struct uverbs_api_object *obj_elm;
        unsigned int i;
        u32 obj_key;
        bool exists;
        int rc;

        obj_key = uapi_key_obj(obj->id);
        obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
        if (IS_ERR(obj_elm))
                return PTR_ERR(obj_elm);

        if (obj->type_attrs) {
                if (WARN_ON(obj_elm->type_attrs))
                        return -EINVAL;

                obj_elm->id = obj->id;
                obj_elm->type_attrs = obj->type_attrs;
                obj_elm->type_class = obj->type_attrs->type_class;
                /*
                 * Today drivers are only permitted to use idr_class and
                 * fd_class types. We can revoke the IDR types during
                 * disassociation, and the FD types require the driver to use
                 * struct file_operations.owner to prevent the driver module
                 * code from unloading while the file is open. This provides
                 * enough safety that uverbs_uobject_fd_release() will
                 * continue to work.  Drivers using FD are responsible to
                 * handle disassociation of the device on their own.
                 */
                if (WARN_ON(is_driver &&
                            obj->type_attrs->type_class != &uverbs_idr_class &&
                            obj->type_attrs->type_class != &uverbs_fd_class))
                        return -EINVAL;
        }

        if (!obj->methods)
                return 0;

        for (i = 0; i != obj->num_methods; i++) {
                const struct uverbs_method_def *method = (*obj->methods)[i];

                if (!method)
                        continue;

                rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
                                       is_driver);
                if (rc)
                        return rc;
        }

        return 0;
}

static int uapi_disable_elm(struct uverbs_api *uapi,
                            const struct uapi_definition *def,
                            u32 obj_key,
                            u32 method_key)
{
        bool exists;

        if (def->scope == UAPI_SCOPE_OBJECT) {
                struct uverbs_api_object *obj_elm;

                obj_elm = uapi_add_get_elm(
                        uapi, obj_key, sizeof(*obj_elm), &exists);
                if (IS_ERR(obj_elm))
                        return PTR_ERR(obj_elm);
                obj_elm->disabled = 1;
                return 0;
        }

        if (def->scope == UAPI_SCOPE_METHOD &&
            uapi_key_is_ioctl_method(method_key)) {
                struct uverbs_api_ioctl_method *method_elm;

                method_elm = uapi_add_get_elm(uapi, method_key,
                                              sizeof(*method_elm), &exists);
                if (IS_ERR(method_elm))
                        return PTR_ERR(method_elm);
                method_elm->disabled = 1;
                return 0;
        }

        if (def->scope == UAPI_SCOPE_METHOD &&
            (uapi_key_is_write_method(method_key) ||
             uapi_key_is_write_ex_method(method_key))) {
                struct uverbs_api_write_method *write_elm;

                write_elm = uapi_add_get_elm(uapi, method_key,
                                             sizeof(*write_elm), &exists);
                if (IS_ERR(write_elm))
                        return PTR_ERR(write_elm);
                write_elm->disabled = 1;
                return 0;
        }

        WARN_ON(true);
        return -EINVAL;
}

static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
                          const struct uapi_definition *def_list,
                          bool is_driver)
{
        const struct uapi_definition *def = def_list;
        u32 cur_obj_key = UVERBS_API_KEY_ERR;
        u32 cur_method_key = UVERBS_API_KEY_ERR;
        bool exists;
        int rc;

        if (!def_list)
                return 0;

        for (;; def++) {
                switch ((enum uapi_definition_kind)def->kind) {
                case UAPI_DEF_CHAIN:
                        rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_CHAIN_OBJ_TREE:
                        if (WARN_ON(def->object_start.object_id !=
                                    def->chain_obj_tree->id))
                                return -EINVAL;

                        cur_obj_key = uapi_key_obj(def->object_start.object_id);
                        rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
                                                 is_driver);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_END:
                        return 0;

                case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
                        void **ibdev_fn =
                                (void *)(&ibdev->ops) + def->needs_fn_offset;

                        if (*ibdev_fn)
                                continue;
                        rc = uapi_disable_elm(
                                uapi, def, cur_obj_key, cur_method_key);
                        if (rc)
                                return rc;
                        continue;
                }

                case UAPI_DEF_IS_SUPPORTED_FUNC:
                        if (def->func_is_supported(ibdev))
                                continue;
                        rc = uapi_disable_elm(
                                uapi, def, cur_obj_key, cur_method_key);
                        if (rc)
                                return rc;
                        continue;

                case UAPI_DEF_OBJECT_START: {
                        struct uverbs_api_object *obj_elm;

                        cur_obj_key = uapi_key_obj(def->object_start.object_id);
                        obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
                                                   sizeof(*obj_elm), &exists);
                        if (IS_ERR(obj_elm))
                                return PTR_ERR(obj_elm);
                        continue;
                }

                case UAPI_DEF_WRITE:
                        rc = uapi_create_write(
                                uapi, ibdev, def, cur_obj_key, &cur_method_key);
                        if (rc)
                                return rc;
                        continue;
                }
                WARN_ON(true);
                return -EINVAL;
        }
}

static int
uapi_finalize_ioctl_method(struct uverbs_api *uapi,
                           struct uverbs_api_ioctl_method *method_elm,
                           u32 method_key)
{
        struct radix_tree_iter iter;
        unsigned int num_attrs = 0;
        unsigned int max_bkey = 0;
        bool single_uobj = false;
        void __rcu **slot;

        method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
        radix_tree_for_each_slot (slot, &uapi->radix, &iter,
                                  uapi_key_attrs_start(method_key)) {
                struct uverbs_api_attr *elm =
                        rcu_dereference_protected(*slot, true);
                u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
                u32 attr_bkey = uapi_bkey_attr(attr_key);
                u8 type = elm->spec.type;

                if (uapi_key_attr_to_ioctl_method(iter.index) !=
                    uapi_key_attr_to_ioctl_method(method_key))
                        break;

                if (elm->spec.mandatory)
                        __set_bit(attr_bkey, method_elm->attr_mandatory);

                if (elm->spec.is_udata)
                        method_elm->has_udata = true;

                if (type == UVERBS_ATTR_TYPE_IDR ||
                    type == UVERBS_ATTR_TYPE_FD) {
                        u8 access = elm->spec.u.obj.access;

                        /*
                         * Verbs specs may only have one NEW/DESTROY, we don't
                         * have the infrastructure to abort multiple NEW's or
                         * cope with multiple DESTROY failure.
                         */
                        if (access == UVERBS_ACCESS_NEW ||
                            access == UVERBS_ACCESS_DESTROY) {
                                if (WARN_ON(single_uobj))
                                        return -EINVAL;

                                single_uobj = true;
                                if (WARN_ON(!elm->spec.mandatory))
                                        return -EINVAL;
                        }

                        if (access == UVERBS_ACCESS_DESTROY)
                                method_elm->destroy_bkey = attr_bkey;
                }

                max_bkey = max(max_bkey, attr_bkey);
                num_attrs++;
        }

        method_elm->key_bitmap_len = max_bkey + 1;
        WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);

        uapi_compute_bundle_size(method_elm, num_attrs);
        return 0;
}

static int uapi_finalize(struct uverbs_api *uapi)
{
        const struct uverbs_api_write_method **data;
        unsigned long max_write_ex = 0;
        unsigned long max_write = 0;
        struct radix_tree_iter iter;
        void __rcu **slot;
        int rc;
        int i;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                struct uverbs_api_ioctl_method *method_elm =
                        rcu_dereference_protected(*slot, true);

                if (uapi_key_is_ioctl_method(iter.index)) {
                        rc = uapi_finalize_ioctl_method(uapi, method_elm,
                                                        iter.index);
                        if (rc)
                                return rc;
                }

                if (uapi_key_is_write_method(iter.index))
                        max_write = max(max_write,
                                        iter.index & UVERBS_API_ATTR_KEY_MASK);
                if (uapi_key_is_write_ex_method(iter.index))
                        max_write_ex =
                                max(max_write_ex,
                                    iter.index & UVERBS_API_ATTR_KEY_MASK);
        }

        uapi->notsupp_method.handler = ib_uverbs_notsupp;
        uapi->num_write = max_write + 1;
        uapi->num_write_ex = max_write_ex + 1;
        data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
                             sizeof(*uapi->write_methods), GFP_KERNEL);
        if (!data)
                return -ENOMEM;

        for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
                data[i] = &uapi->notsupp_method;
        uapi->write_methods = data;
        uapi->write_ex_methods = data + uapi->num_write;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_write_method(iter.index))
                        uapi->write_methods[iter.index &
                                            UVERBS_API_ATTR_KEY_MASK] =
                                rcu_dereference_protected(*slot, true);
                if (uapi_key_is_write_ex_method(iter.index))
                        uapi->write_ex_methods[iter.index &
                                               UVERBS_API_ATTR_KEY_MASK] =
                                rcu_dereference_protected(*slot, true);
        }

        return 0;
}

static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
{
        struct radix_tree_iter iter;
        void __rcu **slot;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
                if (iter.index > last)
                        return;
                kfree(rcu_dereference_protected(*slot, true));
                radix_tree_iter_delete(&uapi->radix, &iter, slot);
        }
}

static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
{
        uapi_remove_range(uapi, obj_key,
                          obj_key | UVERBS_API_METHOD_KEY_MASK |
                                  UVERBS_API_ATTR_KEY_MASK);
}

static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
{
        uapi_remove_range(uapi, method_key,
                          method_key | UVERBS_API_ATTR_KEY_MASK);
}


static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
{
        if (spec->type == UVERBS_ATTR_TYPE_IDR ||
            spec->type == UVERBS_ATTR_TYPE_FD)
                return spec->u.obj.obj_type;
        if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
                return spec->u2.objs_arr.obj_type;
        return UVERBS_API_KEY_ERR;
}

static void uapi_key_okay(u32 key)
{
        unsigned int count = 0;

        if (uapi_key_is_object(key))
                count++;
        if (uapi_key_is_ioctl_method(key))
                count++;
        if (uapi_key_is_write_method(key))
                count++;
        if (uapi_key_is_write_ex_method(key))
                count++;
        if (uapi_key_is_attr(key))
                count++;
        WARN(count != 1, "Bad count %u key=%x", count, key);
}

static void uapi_finalize_disable(struct uverbs_api *uapi)
{
        struct radix_tree_iter iter;
        u32 starting_key = 0;
        bool scan_again = false;
        void __rcu **slot;

again:
        radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
                uapi_key_okay(iter.index);

                if (uapi_key_is_object(iter.index)) {
                        struct uverbs_api_object *obj_elm =
                                rcu_dereference_protected(*slot, true);

                        if (obj_elm->disabled) {
                                /* Have to check all the attrs again */
                                scan_again = true;
                                starting_key = iter.index;
                                uapi_remove_object(uapi, iter.index);
                                goto again;
                        }
                        continue;
                }

                if (uapi_key_is_ioctl_method(iter.index)) {
                        struct uverbs_api_ioctl_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->disabled) {
                                starting_key = iter.index;
                                uapi_remove_method(uapi, iter.index);
                                goto again;
                        }
                        continue;
                }

                if (uapi_key_is_write_method(iter.index) ||
                    uapi_key_is_write_ex_method(iter.index)) {
                        struct uverbs_api_write_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->disabled) {
                                kfree(method_elm);
                                radix_tree_iter_delete(&uapi->radix, &iter, slot);
                        }
                        continue;
                }

                if (uapi_key_is_attr(iter.index)) {
                        struct uverbs_api_attr *attr_elm =
                                rcu_dereference_protected(*slot, true);
                        const struct uverbs_api_object *tmp_obj;
                        u32 obj_key;

                        /*
                         * If the method has a mandatory object handle
                         * attribute which relies on an object which is not
                         * present then the entire method is uncallable.
                         */
                        if (!attr_elm->spec.mandatory)
                                continue;
                        obj_key = uapi_get_obj_id(&attr_elm->spec);
                        if (obj_key == UVERBS_API_KEY_ERR)
                                continue;
                        tmp_obj = uapi_get_object(uapi, obj_key);
                        if (IS_ERR(tmp_obj)) {
                                if (PTR_ERR(tmp_obj) == -ENOMSG)
                                        continue;
                        } else {
                                if (!tmp_obj->disabled)
                                        continue;
                        }

                        starting_key = iter.index;
                        uapi_remove_method(
                                uapi,
                                iter.index & (UVERBS_API_OBJ_KEY_MASK |
                                              UVERBS_API_METHOD_KEY_MASK));
                        goto again;
                }

                WARN_ON(false);
        }

        if (!scan_again)
                return;
        scan_again = false;
        starting_key = 0;
        goto again;
}

void uverbs_destroy_api(struct uverbs_api *uapi)
{
        if (!uapi)
                return;

        uapi_remove_range(uapi, 0, U32_MAX);
        kfree(uapi->write_methods);
        kfree(uapi);
}

static const struct uapi_definition uverbs_core_api[] = {
        UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),
        UAPI_DEF_CHAIN(uverbs_def_obj_counters),
        UAPI_DEF_CHAIN(uverbs_def_obj_cq),
        UAPI_DEF_CHAIN(uverbs_def_obj_device),
        UAPI_DEF_CHAIN(uverbs_def_obj_dm),
        UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
        UAPI_DEF_CHAIN(uverbs_def_obj_intf),
        UAPI_DEF_CHAIN(uverbs_def_obj_mr),
        UAPI_DEF_CHAIN(uverbs_def_obj_qp),
        UAPI_DEF_CHAIN(uverbs_def_obj_srq),
        UAPI_DEF_CHAIN(uverbs_def_obj_wq),
        UAPI_DEF_CHAIN(uverbs_def_write_intf),
        {},
};

struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
{
        struct uverbs_api *uapi;
        int rc;

        uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
        if (!uapi)
                return ERR_PTR(-ENOMEM);

        INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
        uapi->driver_id = ibdev->ops.driver_id;

        rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
        if (rc)
                goto err;
        rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
        if (rc)
                goto err;

        uapi_finalize_disable(uapi);
        rc = uapi_finalize(uapi);
        if (rc)
                goto err;

        return uapi;
err:
        if (rc != -ENOMEM)
                dev_err(&ibdev->dev,
                        "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
                        rc);

        uverbs_destroy_api(uapi);
        return ERR_PTR(rc);
}

/*
 * The pre version is done before destroying the HW objects, it only blocks
 * off method access. All methods that require the ib_dev or the module data
 * must test one of these assignments prior to continuing.
 */
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
{
        struct uverbs_api *uapi = uverbs_dev->uapi;
        struct radix_tree_iter iter;
        void __rcu **slot;

        rcu_assign_pointer(uverbs_dev->ib_dev, NULL);

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_ioctl_method(iter.index)) {
                        struct uverbs_api_ioctl_method *method_elm =
                                rcu_dereference_protected(*slot, true);

                        if (method_elm->driver_method)
                                rcu_assign_pointer(method_elm->handler, NULL);
                }
        }

        synchronize_srcu(&uverbs_dev->disassociate_srcu);
}

/*
 * Called when a driver disassociates from the ib_uverbs_device. The
 * assumption is that the driver module will unload after. Replace everything
 * related to the driver with NULL as a safety measure.
 */
void uverbs_disassociate_api(struct uverbs_api *uapi)
{
        struct radix_tree_iter iter;
        void __rcu **slot;

        radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
                if (uapi_key_is_object(iter.index)) {
                        struct uverbs_api_object *object_elm =
                                rcu_dereference_protected(*slot, true);

                        /*
                         * Some type_attrs are in the driver module. We don't
                         * bother to keep track of which since there should be
                         * no use of this after disassociate.
                         */
                        object_elm->type_attrs = NULL;
                } else if (uapi_key_is_attr(iter.index)) {
                        struct uverbs_api_attr *elm =
                                rcu_dereference_protected(*slot, true);

                        if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
                                elm->spec.u2.enum_def.ids = NULL;
                }
        }
}







































































 2990 







 8067 








 8001 
   71 




























  727 



  333 


  887 
  121 








  759 
  230 




































  512 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#ifndef _LINUX_JHASH_H
#define _LINUX_JHASH_H

/* jhash.h: Jenkins hash support.
 *
 * Copyright (C) 2006. Bob Jenkins (bob_jenkins@burtleburtle.net)
 *
 * https://burtleburtle.net/bob/hash/
 *
 * These are the credits from Bob's sources:
 *
 * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
 *
 * These are functions for producing 32-bit hashes for hash table lookup.
 * hashword(), hashlittle(), hashlittle2(), hashbig(), mix(), and final()
 * are externally useful functions.  Routines to test the hash are included
 * if SELF_TEST is defined.  You can use this free for any purpose.  It's in
 * the public domain.  It has no warranty.
 *
 * Copyright (C) 2009-2010 Jozsef Kadlecsik (kadlec@netfilter.org)
 *
 * I've modified Bob's hash to be useful in the Linux kernel, and
 * any bugs present are my fault.
 * Jozsef
 */
#include <linux/bitops.h>
#include <linux/unaligned.h>

/* Best hash sizes are of power of two */
#define jhash_size(n)   ((u32)1<<(n))
/* Mask the hash value, i.e (value & jhash_mask(n)) instead of (value % n) */
#define jhash_mask(n)   (jhash_size(n)-1)

/* __jhash_mix - mix 3 32-bit values reversibly. */
#define __jhash_mix(a, b, c)                        \
{                                                \
        a -= c;  a ^= rol32(c, 4);  c += b;        \
        b -= a;  b ^= rol32(a, 6);  a += c;        \
        c -= b;  c ^= rol32(b, 8);  b += a;        \
        a -= c;  a ^= rol32(c, 16); c += b;        \
        b -= a;  b ^= rol32(a, 19); a += c;        \
        c -= b;  c ^= rol32(b, 4);  b += a;        \
}

/* __jhash_final - final mixing of 3 32-bit values (a,b,c) into c */
#define __jhash_final(a, b, c)                        \
{                                                \
        c ^= b; c -= rol32(b, 14);                \
        a ^= c; a -= rol32(c, 11);                \
        b ^= a; b -= rol32(a, 25);                \
        c ^= b; c -= rol32(b, 16);                \
        a ^= c; a -= rol32(c, 4);                \
        b ^= a; b -= rol32(a, 14);                \
        c ^= b; c -= rol32(b, 24);                \
}

/* An arbitrary initial parameter */
#define JHASH_INITVAL                0xdeadbeef

/* jhash - hash an arbitrary key
 * @k: sequence of bytes as key
 * @length: the length of the key
 * @initval: the previous hash, or an arbitrary value
 *
 * The generic version, hashes an arbitrary sequence of bytes.
 * No alignment or length assumptions are made about the input key.
 *
 * Returns the hash value of the key. The result depends on endianness.
 */
static inline u32 jhash(const void *key, u32 length, u32 initval)
{
        u32 a, b, c;
        const u8 *k = key;

        /* Set up the internal state */
        a = b = c = JHASH_INITVAL + length + initval;

        /* All but the last block: affect some 32 bits of (a,b,c) */
        while (length > 12) {
                a += get_unaligned((u32 *)k);
                b += get_unaligned((u32 *)(k + 4));
                c += get_unaligned((u32 *)(k + 8));
                __jhash_mix(a, b, c);
                length -= 12;
                k += 12;
        }
        /* Last block: affect all 32 bits of (c) */
        switch (length) {
        case 12: c += (u32)k[11]<<24;        fallthrough;
        case 11: c += (u32)k[10]<<16;        fallthrough;
        case 10: c += (u32)k[9]<<8;        fallthrough;
        case 9:  c += k[8];                fallthrough;
        case 8:  b += (u32)k[7]<<24;        fallthrough;
        case 7:  b += (u32)k[6]<<16;        fallthrough;
        case 6:  b += (u32)k[5]<<8;        fallthrough;
        case 5:  b += k[4];                fallthrough;
        case 4:  a += (u32)k[3]<<24;        fallthrough;
        case 3:  a += (u32)k[2]<<16;        fallthrough;
        case 2:  a += (u32)k[1]<<8;        fallthrough;
        case 1:  a += k[0];
                 __jhash_final(a, b, c);
                 break;
        case 0: /* Nothing left to add */
                break;
        }

        return c;
}

/* jhash2 - hash an array of u32's
 * @k: the key which must be an array of u32's
 * @length: the number of u32's in the key
 * @initval: the previous hash, or an arbitrary value
 *
 * Returns the hash value of the key.
 */
static inline u32 jhash2(const u32 *k, u32 length, u32 initval)
{
        u32 a, b, c;

        /* Set up the internal state */
        a = b = c = JHASH_INITVAL + (length<<2) + initval;

        /* Handle most of the key */
        while (length > 3) {
                a += k[0];
                b += k[1];
                c += k[2];
                __jhash_mix(a, b, c);
                length -= 3;
                k += 3;
        }

        /* Handle the last 3 u32's */
        switch (length) {
        case 3: c += k[2];        fallthrough;
        case 2: b += k[1];        fallthrough;
        case 1: a += k[0];
                __jhash_final(a, b, c);
                break;
        case 0:        /* Nothing left to add */
                break;
        }

        return c;
}


/* __jhash_nwords - hash exactly 3, 2 or 1 word(s) */
static inline u32 __jhash_nwords(u32 a, u32 b, u32 c, u32 initval)
{
        a += initval;
        b += initval;
        c += initval;

        __jhash_final(a, b, c);

        return c;
}

static inline u32 jhash_3words(u32 a, u32 b, u32 c, u32 initval)
{
        return __jhash_nwords(a, b, c, initval + JHASH_INITVAL + (3 << 2));
}

static inline u32 jhash_2words(u32 a, u32 b, u32 initval)
{
        return __jhash_nwords(a, b, 0, initval + JHASH_INITVAL + (2 << 2));
}

static inline u32 jhash_1word(u32 a, u32 initval)
{
        return __jhash_nwords(a, 0, 0, initval + JHASH_INITVAL + (1 << 2));
}

#endif /* _LINUX_JHASH_H */




























    2 




























































































































































    2 










































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 








    2 






    2 




















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008, 2009 open80211s Ltd.
 * Copyright (C) 2018 - 2024 Intel Corporation
 * Authors:    Luis Carlos Cobo <luisca@cozybit.com>
 *                Javier Cardona <javier@cozybit.com>
 */

#include <linux/slab.h>
#include <linux/unaligned.h>
#include <net/sock.h>
#include "ieee80211_i.h"
#include "mesh.h"
#include "wme.h"
#include "driver-ops.h"

static int mesh_allocated;
static struct kmem_cache *rm_cache;

bool mesh_action_is_path_sel(struct ieee80211_mgmt *mgmt)
{
        return (mgmt->u.action.u.mesh_action.action_code ==
                        WLAN_MESH_ACTION_HWMP_PATH_SELECTION);
}

void ieee80211s_init(void)
{
        mesh_allocated = 1;
        rm_cache = kmem_cache_create("mesh_rmc", sizeof(struct rmc_entry),
                                     0, 0, NULL);
}

void ieee80211s_stop(void)
{
        if (!mesh_allocated)
                return;
        kmem_cache_destroy(rm_cache);
}

static void ieee80211_mesh_housekeeping_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.housekeeping_timer);
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        set_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags);

        wiphy_work_queue(local->hw.wiphy, &sdata->work);
}

/**
 * mesh_matches_local - check if the config of a mesh point matches ours
 *
 * @sdata: local mesh subif
 * @ie: information elements of a management frame from the mesh peer
 *
 * This function checks if the mesh configuration of a mesh point matches the
 * local mesh configuration, i.e. if both nodes belong to the same mesh network.
 *
 * Returns: %true if both nodes belong to the same mesh
 */
bool mesh_matches_local(struct ieee80211_sub_if_data *sdata,
                        struct ieee802_11_elems *ie)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 basic_rates = 0;
        struct cfg80211_chan_def sta_chan_def;
        struct ieee80211_supported_band *sband;
        u32 vht_cap_info = 0;

        /*
         * As support for each feature is added, check for matching
         * - On mesh config capabilities
         *   - Power Save Support En
         *   - Sync support enabled
         *   - Sync support active
         *   - Sync support required from peer
         *   - MDA enabled
         * - Power management control on fc
         */
        if (!(ifmsh->mesh_id_len == ie->mesh_id_len &&
             memcmp(ifmsh->mesh_id, ie->mesh_id, ie->mesh_id_len) == 0 &&
             (ifmsh->mesh_pp_id == ie->mesh_config->meshconf_psel) &&
             (ifmsh->mesh_pm_id == ie->mesh_config->meshconf_pmetric) &&
             (ifmsh->mesh_cc_id == ie->mesh_config->meshconf_congest) &&
             (ifmsh->mesh_sp_id == ie->mesh_config->meshconf_synch) &&
             (ifmsh->mesh_auth_id == ie->mesh_config->meshconf_auth)))
                return false;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return false;

        ieee80211_sta_get_rates(sdata, ie, sband->band,
                                &basic_rates);

        if (sdata->vif.bss_conf.basic_rates != basic_rates)
                return false;

        cfg80211_chandef_create(&sta_chan_def, sdata->vif.bss_conf.chanreq.oper.chan,
                                NL80211_CHAN_NO_HT);
        ieee80211_chandef_ht_oper(ie->ht_operation, &sta_chan_def);

        if (ie->vht_cap_elem)
                vht_cap_info = le32_to_cpu(ie->vht_cap_elem->vht_cap_info);

        ieee80211_chandef_vht_oper(&sdata->local->hw, vht_cap_info,
                                   ie->vht_operation, ie->ht_operation,
                                   &sta_chan_def);
        ieee80211_chandef_he_6ghz_oper(sdata->local, ie->he_operation,
                                       ie->eht_operation,
                                       &sta_chan_def);

        if (!cfg80211_chandef_compatible(&sdata->vif.bss_conf.chanreq.oper,
                                         &sta_chan_def))
                return false;

        return true;
}

/**
 * mesh_peer_accepts_plinks - check if an mp is willing to establish peer links
 *
 * @ie: information elements of a management frame from the mesh peer
 *
 * Returns: %true if the mesh peer is willing to establish peer links
 */
bool mesh_peer_accepts_plinks(struct ieee802_11_elems *ie)
{
        return (ie->mesh_config->meshconf_cap &
                        IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS) != 0;
}

/**
 * mesh_accept_plinks_update - update accepting_plink in local mesh beacons
 *
 * @sdata: mesh interface in which mesh beacons are going to be updated
 *
 * Returns: beacon changed flag if the beacon content changed.
 */
u64 mesh_accept_plinks_update(struct ieee80211_sub_if_data *sdata)
{
        bool free_plinks;
        u64 changed = 0;

        /* In case mesh_plink_free_count > 0 and mesh_plinktbl_capacity == 0,
         * the mesh interface might be able to establish plinks with peers that
         * are already on the table but are not on PLINK_ESTAB state. However,
         * in general the mesh interface is not accepting peer link requests
         * from new peers, and that must be reflected in the beacon
         */
        free_plinks = mesh_plink_availables(sdata);

        if (free_plinks != sdata->u.mesh.accepting_plinks) {
                sdata->u.mesh.accepting_plinks = free_plinks;
                changed = BSS_CHANGED_BEACON;
        }

        return changed;
}

/*
 * mesh_sta_cleanup - clean up any mesh sta state
 *
 * @sta: mesh sta to clean up.
 */
void mesh_sta_cleanup(struct sta_info *sta)
{
        struct ieee80211_sub_if_data *sdata = sta->sdata;
        u64 changed = mesh_plink_deactivate(sta);

        if (changed)
                ieee80211_mbss_info_change_notify(sdata, changed);
}

int mesh_rmc_init(struct ieee80211_sub_if_data *sdata)
{
        int i;

        sdata->u.mesh.rmc = kmalloc(sizeof(struct mesh_rmc), GFP_KERNEL);
        if (!sdata->u.mesh.rmc)
                return -ENOMEM;
        sdata->u.mesh.rmc->idx_mask = RMC_BUCKETS - 1;
        for (i = 0; i < RMC_BUCKETS; i++)
                INIT_HLIST_HEAD(&sdata->u.mesh.rmc->bucket[i]);
        return 0;
}

void mesh_rmc_free(struct ieee80211_sub_if_data *sdata)
{
        struct mesh_rmc *rmc = sdata->u.mesh.rmc;
        struct rmc_entry *p;
        struct hlist_node *n;
        int i;

        if (!sdata->u.mesh.rmc)
                return;

        for (i = 0; i < RMC_BUCKETS; i++) {
                hlist_for_each_entry_safe(p, n, &rmc->bucket[i], list) {
                        hlist_del(&p->list);
                        kmem_cache_free(rm_cache, p);
                }
        }

        kfree(rmc);
        sdata->u.mesh.rmc = NULL;
}

/**
 * mesh_rmc_check - Check frame in recent multicast cache and add if absent.
 *
 * @sdata:        interface
 * @sa:                source address
 * @mesh_hdr:        mesh_header
 *
 * Returns: 0 if the frame is not in the cache, nonzero otherwise.
 *
 * Checks using the source address and the mesh sequence number if we have
 * received this frame lately. If the frame is not in the cache, it is added to
 * it.
 */
int mesh_rmc_check(struct ieee80211_sub_if_data *sdata,
                   const u8 *sa, struct ieee80211s_hdr *mesh_hdr)
{
        struct mesh_rmc *rmc = sdata->u.mesh.rmc;
        u32 seqnum = 0;
        int entries = 0;
        u8 idx;
        struct rmc_entry *p;
        struct hlist_node *n;

        if (!rmc)
                return -1;

        /* Don't care about endianness since only match matters */
        memcpy(&seqnum, &mesh_hdr->seqnum, sizeof(mesh_hdr->seqnum));
        idx = le32_to_cpu(mesh_hdr->seqnum) & rmc->idx_mask;
        hlist_for_each_entry_safe(p, n, &rmc->bucket[idx], list) {
                ++entries;
                if (time_after(jiffies, p->exp_time) ||
                    entries == RMC_QUEUE_MAX_LEN) {
                        hlist_del(&p->list);
                        kmem_cache_free(rm_cache, p);
                        --entries;
                } else if ((seqnum == p->seqnum) && ether_addr_equal(sa, p->sa))
                        return -1;
        }

        p = kmem_cache_alloc(rm_cache, GFP_ATOMIC);
        if (!p)
                return 0;

        p->seqnum = seqnum;
        p->exp_time = jiffies + RMC_TIMEOUT;
        memcpy(p->sa, sa, ETH_ALEN);
        hlist_add_head(&p->list, &rmc->bucket[idx]);
        return 0;
}

int mesh_add_meshconf_ie(struct ieee80211_sub_if_data *sdata,
                         struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos, neighbors;
        u8 meshconf_len = sizeof(struct ieee80211_meshconf_ie);
        bool is_connected_to_gate = ifmsh->num_gates > 0 ||
                ifmsh->mshcfg.dot11MeshGateAnnouncementProtocol ||
                ifmsh->mshcfg.dot11MeshConnectedToMeshGate;
        bool is_connected_to_as = ifmsh->mshcfg.dot11MeshConnectedToAuthServer;

        if (skb_tailroom(skb) < 2 + meshconf_len)
                return -ENOMEM;

        pos = skb_put(skb, 2 + meshconf_len);
        *pos++ = WLAN_EID_MESH_CONFIG;
        *pos++ = meshconf_len;

        /* save a pointer for quick updates in pre-tbtt */
        ifmsh->meshconf_offset = pos - skb->data;

        /* Active path selection protocol ID */
        *pos++ = ifmsh->mesh_pp_id;
        /* Active path selection metric ID   */
        *pos++ = ifmsh->mesh_pm_id;
        /* Congestion control mode identifier */
        *pos++ = ifmsh->mesh_cc_id;
        /* Synchronization protocol identifier */
        *pos++ = ifmsh->mesh_sp_id;
        /* Authentication Protocol identifier */
        *pos++ = ifmsh->mesh_auth_id;
        /* Mesh Formation Info - number of neighbors */
        neighbors = atomic_read(&ifmsh->estab_plinks);
        neighbors = min_t(int, neighbors, IEEE80211_MAX_MESH_PEERINGS);
        *pos++ = (is_connected_to_as << 7) |
                 (neighbors << 1) |
                 is_connected_to_gate;
        /* Mesh capability */
        *pos = 0x00;
        *pos |= ifmsh->mshcfg.dot11MeshForwarding ?
                        IEEE80211_MESHCONF_CAPAB_FORWARDING : 0x00;
        *pos |= ifmsh->accepting_plinks ?
                        IEEE80211_MESHCONF_CAPAB_ACCEPT_PLINKS : 0x00;
        /* Mesh PS mode. See IEEE802.11-2012 8.4.2.100.8 */
        *pos |= ifmsh->ps_peers_deep_sleep ?
                        IEEE80211_MESHCONF_CAPAB_POWER_SAVE_LEVEL : 0x00;
        return 0;
}

int mesh_add_meshid_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos;

        if (skb_tailroom(skb) < 2 + ifmsh->mesh_id_len)
                return -ENOMEM;

        pos = skb_put(skb, 2 + ifmsh->mesh_id_len);
        *pos++ = WLAN_EID_MESH_ID;
        *pos++ = ifmsh->mesh_id_len;
        if (ifmsh->mesh_id_len)
                memcpy(pos, ifmsh->mesh_id, ifmsh->mesh_id_len);

        return 0;
}

static int mesh_add_awake_window_ie(struct ieee80211_sub_if_data *sdata,
                                    struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 *pos;

        /* see IEEE802.11-2012 13.14.6 */
        if (ifmsh->ps_peers_light_sleep == 0 &&
            ifmsh->ps_peers_deep_sleep == 0 &&
            ifmsh->nonpeer_pm == NL80211_MESH_POWER_ACTIVE)
                return 0;

        if (skb_tailroom(skb) < 4)
                return -ENOMEM;

        pos = skb_put(skb, 2 + 2);
        *pos++ = WLAN_EID_MESH_AWAKE_WINDOW;
        *pos++ = 2;
        put_unaligned_le16(ifmsh->mshcfg.dot11MeshAwakeWindowDuration, pos);

        return 0;
}

int mesh_add_vendor_ies(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 offset, len;
        const u8 *data;

        if (!ifmsh->ie || !ifmsh->ie_len)
                return 0;

        /* fast-forward to vendor IEs */
        offset = ieee80211_ie_split_vendor(ifmsh->ie, ifmsh->ie_len, 0);

        if (offset < ifmsh->ie_len) {
                len = ifmsh->ie_len - offset;
                data = ifmsh->ie + offset;
                if (skb_tailroom(skb) < len)
                        return -ENOMEM;
                skb_put_data(skb, data, len);
        }

        return 0;
}

int mesh_add_rsn_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u8 len = 0;
        const u8 *data;

        if (!ifmsh->ie || !ifmsh->ie_len)
                return 0;

        /* find RSN IE */
        data = cfg80211_find_ie(WLAN_EID_RSN, ifmsh->ie, ifmsh->ie_len);
        if (!data)
                return 0;

        len = data[1] + 2;

        if (skb_tailroom(skb) < len)
                return -ENOMEM;
        skb_put_data(skb, data, len);

        return 0;
}

static int mesh_add_ds_params_ie(struct ieee80211_sub_if_data *sdata,
                                 struct sk_buff *skb)
{
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *chan;
        u8 *pos;

        if (skb_tailroom(skb) < 3)
                return -ENOMEM;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        chan = chanctx_conf->def.chan;
        rcu_read_unlock();

        pos = skb_put(skb, 2 + 1);
        *pos++ = WLAN_EID_DS_PARAMS;
        *pos++ = 1;
        *pos++ = ieee80211_frequency_to_channel(chan->center_freq);

        return 0;
}

int mesh_add_ht_cap_ie(struct ieee80211_sub_if_data *sdata,
                       struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        /* HT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!sband->ht_cap.ht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_ht_cap))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_cap));
        ieee80211_ie_build_ht_cap(pos, &sband->ht_cap, sband->ht_cap.cap);

        return 0;
}

int mesh_add_ht_oper_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *channel;
        struct ieee80211_supported_band *sband;
        struct ieee80211_sta_ht_cap *ht_cap;
        u8 *pos;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        channel = chanctx_conf->def.chan;
        rcu_read_unlock();

        sband = local->hw.wiphy->bands[channel->band];
        ht_cap = &sband->ht_cap;

        /* HT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!ht_cap->ht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_ht_operation))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_ht_operation));
        ieee80211_ie_build_ht_oper(pos, ht_cap, &sdata->vif.bss_conf.chanreq.oper,
                                   sdata->vif.bss_conf.ht_operation_mode,
                                   false);

        return 0;
}

int mesh_add_vht_cap_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        /* VHT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!sband->vht_cap.vht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_vht_cap))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_cap));
        ieee80211_ie_build_vht_cap(pos, &sband->vht_cap, sband->vht_cap.cap);

        return 0;
}

int mesh_add_vht_oper_ie(struct ieee80211_sub_if_data *sdata,
                         struct sk_buff *skb)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_chanctx_conf *chanctx_conf;
        struct ieee80211_channel *channel;
        struct ieee80211_supported_band *sband;
        struct ieee80211_sta_vht_cap *vht_cap;
        u8 *pos;

        rcu_read_lock();
        chanctx_conf = rcu_dereference(sdata->vif.bss_conf.chanctx_conf);
        if (WARN_ON(!chanctx_conf)) {
                rcu_read_unlock();
                return -EINVAL;
        }
        channel = chanctx_conf->def.chan;
        rcu_read_unlock();

        sband = local->hw.wiphy->bands[channel->band];
        vht_cap = &sband->vht_cap;

        /* VHT not allowed in 6 GHz */
        if (sband->band == NL80211_BAND_6GHZ)
                return 0;

        if (!vht_cap->vht_supported ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        if (skb_tailroom(skb) < 2 + sizeof(struct ieee80211_vht_operation))
                return -ENOMEM;

        pos = skb_put(skb, 2 + sizeof(struct ieee80211_vht_operation));
        ieee80211_ie_build_vht_oper(pos, vht_cap,
                                    &sdata->vif.bss_conf.chanreq.oper);

        return 0;
}

int mesh_add_he_cap_ie(struct ieee80211_sub_if_data *sdata,
                       struct sk_buff *skb, u8 ie_len)
{
        struct ieee80211_supported_band *sband;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        if (sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        return ieee80211_put_he_cap(skb, sdata, sband, NULL);
}

int mesh_add_he_oper_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb)
{
        const struct ieee80211_sta_he_cap *he_cap;
        struct ieee80211_supported_band *sband;
        u32 len;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        he_cap = ieee80211_get_he_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
        if (!he_cap ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        len = 2 + 1 + sizeof(struct ieee80211_he_operation);
        if (sdata->vif.bss_conf.chanreq.oper.chan->band == NL80211_BAND_6GHZ)
                len += sizeof(struct ieee80211_he_6ghz_oper);

        if (skb_tailroom(skb) < len)
                return -ENOMEM;

        pos = skb_put(skb, len);
        ieee80211_ie_build_he_oper(pos, &sdata->vif.bss_conf.chanreq.oper);

        return 0;
}

int mesh_add_he_6ghz_cap_ie(struct ieee80211_sub_if_data *sdata,
                            struct sk_buff *skb)
{
        struct ieee80211_supported_band *sband;
        const struct ieee80211_sband_iftype_data *iftd;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        iftd = ieee80211_get_sband_iftype_data(sband,
                                               NL80211_IFTYPE_MESH_POINT);
        /* The device doesn't support HE in mesh mode or at all */
        if (!iftd)
                return 0;

        ieee80211_put_he_6ghz_cap(skb, sdata, sdata->deflink.smps_mode);
        return 0;
}

int mesh_add_eht_cap_ie(struct ieee80211_sub_if_data *sdata,
                        struct sk_buff *skb, u8 ie_len)
{
        struct ieee80211_supported_band *sband;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        if (sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        return ieee80211_put_eht_cap(skb, sdata, sband, NULL);
}

int mesh_add_eht_oper_ie(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb)
{
        const struct ieee80211_sta_eht_cap *eht_cap;
        struct ieee80211_supported_band *sband;
        u32 len;
        u8 *pos;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return -EINVAL;

        eht_cap = ieee80211_get_eht_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
        if (!eht_cap ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return 0;

        len = 2 + 1 + offsetof(struct ieee80211_eht_operation, optional) +
                      offsetof(struct ieee80211_eht_operation_info, optional);

        if (skb_tailroom(skb) < len)
                return -ENOMEM;

        pos = skb_put(skb, len);
        ieee80211_ie_build_eht_oper(pos, &sdata->vif.bss_conf.chanreq.oper, eht_cap);

        return 0;
}

static void ieee80211_mesh_path_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.mesh_path_timer);

        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

static void ieee80211_mesh_path_root_timer(struct timer_list *t)
{
        struct ieee80211_sub_if_data *sdata =
                timer_container_of(sdata, t, u.mesh.mesh_path_root_timer);
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        set_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);

        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

void ieee80211_mesh_root_setup(struct ieee80211_if_mesh *ifmsh)
{
        if (ifmsh->mshcfg.dot11MeshHWMPRootMode > IEEE80211_ROOTMODE_ROOT)
                set_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);
        else {
                clear_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags);
                /* stop running timer */
                timer_delete_sync(&ifmsh->mesh_path_root_timer);
        }
}

static void
ieee80211_mesh_update_bss_params(struct ieee80211_sub_if_data *sdata,
                                 u8 *ie, u8 ie_len)
{
        struct ieee80211_supported_band *sband;
        const struct element *cap;
        const struct ieee80211_he_operation *he_oper = NULL;

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return;

        if (!ieee80211_get_he_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT) ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_20_NOHT ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_5 ||
            sdata->vif.bss_conf.chanreq.oper.width == NL80211_CHAN_WIDTH_10)
                return;

        sdata->vif.bss_conf.he_support = true;

        cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ie, ie_len);
        if (cap && cap->datalen >= 1 + sizeof(*he_oper) &&
            cap->datalen >= 1 + ieee80211_he_oper_size(cap->data + 1))
                he_oper = (void *)(cap->data + 1);

        if (he_oper)
                sdata->vif.bss_conf.he_oper.params =
                        __le32_to_cpu(he_oper->he_oper_params);

        sdata->vif.bss_conf.eht_support =
                !!ieee80211_get_eht_iftype_cap(sband, NL80211_IFTYPE_MESH_POINT);
}

bool ieee80211_mesh_xmit_fast(struct ieee80211_sub_if_data *sdata,
                              struct sk_buff *skb, u32 ctrl_flags)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_mesh_fast_tx_key key = {
                .type = MESH_FAST_TX_TYPE_LOCAL
        };
        struct ieee80211_mesh_fast_tx *entry;
        struct ieee80211s_hdr *meshhdr;
        u8 sa[ETH_ALEN] __aligned(2);
        struct tid_ampdu_tx *tid_tx;
        struct sta_info *sta;
        bool copy_sa = false;
        u16 ethertype;
        u8 tid;

        if (ctrl_flags & IEEE80211_TX_CTRL_SKIP_MPATH_LOOKUP)
                return false;

        if (ifmsh->mshcfg.dot11MeshNolearn)
                return false;

        /* Add support for these cases later */
        if (ifmsh->ps_peers_light_sleep || ifmsh->ps_peers_deep_sleep)
                return false;

        if (is_multicast_ether_addr(skb->data))
                return false;

        ethertype = (skb->data[12] << 8) | skb->data[13];
        if (ethertype < ETH_P_802_3_MIN)
                return false;

        if (sk_requests_wifi_status(skb->sk))
                return false;

        if (skb->ip_summed == CHECKSUM_PARTIAL) {
                skb_set_transport_header(skb, skb_checksum_start_offset(skb));
                if (skb_checksum_help(skb))
                        return false;
        }

        ether_addr_copy(key.addr, skb->data);
        if (!ether_addr_equal(skb->data + ETH_ALEN, sdata->vif.addr))
                key.type = MESH_FAST_TX_TYPE_PROXIED;
        entry = mesh_fast_tx_get(sdata, &key);
        if (!entry)
                return false;

        if (skb_headroom(skb) < entry->hdrlen + entry->fast_tx.hdr_len)
                return false;

        sta = rcu_dereference(entry->mpath->next_hop);
        if (!sta)
                return false;

        tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK;
        tid_tx = rcu_dereference(sta->ampdu_mlme.tid_tx[tid]);
        if (tid_tx) {
                if (!test_bit(HT_AGG_STATE_OPERATIONAL, &tid_tx->state))
                        return false;
                if (tid_tx->timeout)
                        tid_tx->last_tx = jiffies;
        }

        skb = skb_share_check(skb, GFP_ATOMIC);
        if (!skb)
                return true;

        skb_set_queue_mapping(skb, ieee80211_select_queue(sdata, sta, skb));

        meshhdr = (struct ieee80211s_hdr *)entry->hdr;
        if ((meshhdr->flags & MESH_FLAGS_AE) == MESH_FLAGS_AE_A5_A6) {
                /* preserve SA from eth header for 6-addr frames */
                ether_addr_copy(sa, skb->data + ETH_ALEN);
                copy_sa = true;
        }

        memcpy(skb_push(skb, entry->hdrlen - 2 * ETH_ALEN), entry->hdr,
               entry->hdrlen);

        meshhdr = (struct ieee80211s_hdr *)skb->data;
        put_unaligned_le32(atomic_inc_return(&sdata->u.mesh.mesh_seqnum),
                           &meshhdr->seqnum);
        meshhdr->ttl = sdata->u.mesh.mshcfg.dot11MeshTTL;
        if (copy_sa)
            ether_addr_copy(meshhdr->eaddr2, sa);

        skb_push(skb, 2 * ETH_ALEN);
        __ieee80211_xmit_fast(sdata, sta, &entry->fast_tx, skb, tid_tx,
                              entry->mpath->dst, sdata->vif.addr);

        return true;
}

/**
 * ieee80211_fill_mesh_addresses - fill addresses of a locally originated mesh frame
 * @hdr:        802.11 frame header
 * @fc:                frame control field
 * @meshda:        destination address in the mesh
 * @meshsa:        source address in the mesh.  Same as TA, as frame is
 *              locally originated.
 *
 * Returns: the length of the 802.11 frame header (excludes mesh control header)
 */
int ieee80211_fill_mesh_addresses(struct ieee80211_hdr *hdr, __le16 *fc,
                                  const u8 *meshda, const u8 *meshsa)
{
        if (is_multicast_ether_addr(meshda)) {
                *fc |= cpu_to_le16(IEEE80211_FCTL_FROMDS);
                /* DA TA SA */
                memcpy(hdr->addr1, meshda, ETH_ALEN);
                memcpy(hdr->addr2, meshsa, ETH_ALEN);
                memcpy(hdr->addr3, meshsa, ETH_ALEN);
                return 24;
        } else {
                *fc |= cpu_to_le16(IEEE80211_FCTL_FROMDS | IEEE80211_FCTL_TODS);
                /* RA TA DA SA */
                eth_zero_addr(hdr->addr1);   /* RA is resolved later */
                memcpy(hdr->addr2, meshsa, ETH_ALEN);
                memcpy(hdr->addr3, meshda, ETH_ALEN);
                memcpy(hdr->addr4, meshsa, ETH_ALEN);
                return 30;
        }
}

/**
 * ieee80211_new_mesh_header - create a new mesh header
 * @sdata:        mesh interface to be used
 * @meshhdr:    uninitialized mesh header
 * @addr4or5:   1st address in the ae header, which may correspond to address 4
 *              (if addr6 is NULL) or address 5 (if addr6 is present). It may
 *              be NULL.
 * @addr6:        2nd address in the ae header, which corresponds to addr6 of the
 *              mesh frame
 *
 * Returns: the header length
 */
unsigned int ieee80211_new_mesh_header(struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211s_hdr *meshhdr,
                                       const char *addr4or5, const char *addr6)
{
        if (WARN_ON(!addr4or5 && addr6))
                return 0;

        memset(meshhdr, 0, sizeof(*meshhdr));

        meshhdr->ttl = sdata->u.mesh.mshcfg.dot11MeshTTL;

        put_unaligned_le32(atomic_inc_return(&sdata->u.mesh.mesh_seqnum),
                           &meshhdr->seqnum);
        if (addr4or5 && !addr6) {
                meshhdr->flags |= MESH_FLAGS_AE_A4;
                memcpy(meshhdr->eaddr1, addr4or5, ETH_ALEN);
                return 2 * ETH_ALEN;
        } else if (addr4or5 && addr6) {
                meshhdr->flags |= MESH_FLAGS_AE_A5_A6;
                memcpy(meshhdr->eaddr1, addr4or5, ETH_ALEN);
                memcpy(meshhdr->eaddr2, addr6, ETH_ALEN);
                return 3 * ETH_ALEN;
        }

        return ETH_ALEN;
}

static void ieee80211_mesh_housekeeping(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u64 changed;

        if (ifmsh->mshcfg.plink_timeout > 0)
                ieee80211_sta_expire(sdata, ifmsh->mshcfg.plink_timeout * HZ);
        mesh_path_expire(sdata);

        changed = mesh_accept_plinks_update(sdata);
        ieee80211_mbss_info_change_notify(sdata, changed);

        mesh_fast_tx_gc(sdata);

        mod_timer(&ifmsh->housekeeping_timer,
                  round_jiffies(jiffies +
                                IEEE80211_MESH_HOUSEKEEPING_INTERVAL));
}

static void ieee80211_mesh_rootpath(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 interval;

        mesh_path_tx_root_frame(sdata);

        if (ifmsh->mshcfg.dot11MeshHWMPRootMode == IEEE80211_PROACTIVE_RANN)
                interval = ifmsh->mshcfg.dot11MeshHWMPRannInterval;
        else
                interval = ifmsh->mshcfg.dot11MeshHWMProotInterval;

        mod_timer(&ifmsh->mesh_path_root_timer,
                  round_jiffies(TU_TO_EXP_TIME(interval)));
}

static int
ieee80211_mesh_build_beacon(struct ieee80211_if_mesh *ifmsh)
{
        struct beacon_data *bcn;
        int head_len, tail_len;
        struct sk_buff *skb;
        struct ieee80211_mgmt *mgmt;
        struct mesh_csa_settings *csa;
        const struct ieee80211_supported_band *sband;
        u8 ie_len_he_cap, ie_len_eht_cap;
        u8 *pos;
        struct ieee80211_sub_if_data *sdata;
        int hdr_len = offsetofend(struct ieee80211_mgmt, u.beacon);

        sdata = container_of(ifmsh, struct ieee80211_sub_if_data, u.mesh);

        sband = ieee80211_get_sband(sdata);

        ie_len_he_cap = ieee80211_ie_len_he_cap(sdata);
        ie_len_eht_cap = ieee80211_ie_len_eht_cap(sdata);
        head_len = hdr_len +
                   2 + /* NULL SSID */
                   /* Channel Switch Announcement */
                   2 + sizeof(struct ieee80211_channel_sw_ie) +
                   /* Mesh Channel Switch Parameters */
                   2 + sizeof(struct ieee80211_mesh_chansw_params_ie) +
                   /* Channel Switch Wrapper + Wide Bandwidth CSA IE */
                   2 + 2 + sizeof(struct ieee80211_wide_bw_chansw_ie) +
                   2 + sizeof(struct ieee80211_sec_chan_offs_ie) +
                   2 + 8 + /* supported rates */
                   2 + 3; /* DS params */
        tail_len = 2 + (IEEE80211_MAX_SUPP_RATES - 8) +
                   2 + sizeof(struct ieee80211_ht_cap) +
                   2 + sizeof(struct ieee80211_ht_operation) +
                   2 + ifmsh->mesh_id_len +
                   2 + sizeof(struct ieee80211_meshconf_ie) +
                   2 + sizeof(__le16) + /* awake window */
                   2 + sizeof(struct ieee80211_vht_cap) +
                   2 + sizeof(struct ieee80211_vht_operation) +
                   ie_len_he_cap +
                   2 + 1 + sizeof(struct ieee80211_he_operation) +
                           sizeof(struct ieee80211_he_6ghz_oper) +
                   2 + 1 + sizeof(struct ieee80211_he_6ghz_capa) +
                   ie_len_eht_cap +
                   2 + 1 + offsetof(struct ieee80211_eht_operation, optional) +
                           offsetof(struct ieee80211_eht_operation_info, optional) +
                   ifmsh->ie_len;

        bcn = kzalloc(sizeof(*bcn) + head_len + tail_len, GFP_KERNEL);
        /* need an skb for IE builders to operate on */
        skb = __dev_alloc_skb(max(head_len, tail_len), GFP_KERNEL);

        if (!bcn || !skb)
                goto out_free;

        /*
         * pointers go into the block we allocated,
         * memory is | beacon_data | head | tail |
         */
        bcn->head = ((u8 *) bcn) + sizeof(*bcn);

        /* fill in the head */
        mgmt = skb_put_zero(skb, hdr_len);
        mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                          IEEE80211_STYPE_BEACON);
        eth_broadcast_addr(mgmt->da);
        memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt->bssid, sdata->vif.addr, ETH_ALEN);
        ieee80211_mps_set_frame_flags(sdata, NULL, (void *) mgmt);
        mgmt->u.beacon.beacon_int =
                cpu_to_le16(sdata->vif.bss_conf.beacon_int);
        mgmt->u.beacon.capab_info |= cpu_to_le16(
                sdata->u.mesh.security ? WLAN_CAPABILITY_PRIVACY : 0);

        pos = skb_put(skb, 2);
        *pos++ = WLAN_EID_SSID;
        *pos++ = 0x0;

        rcu_read_lock();
        csa = rcu_dereference(ifmsh->csa);
        if (csa) {
                enum nl80211_channel_type ct;
                struct cfg80211_chan_def *chandef;
                int ie_len = 2 + sizeof(struct ieee80211_channel_sw_ie) +
                             2 + sizeof(struct ieee80211_mesh_chansw_params_ie);

                pos = skb_put_zero(skb, ie_len);
                *pos++ = WLAN_EID_CHANNEL_SWITCH;
                *pos++ = 3;
                *pos++ = 0x0;
                *pos++ = ieee80211_frequency_to_channel(
                                csa->settings.chandef.chan->center_freq);
                bcn->cntdwn_current_counter = csa->settings.count;
                bcn->cntdwn_counter_offsets[0] = hdr_len + 6;
                *pos++ = csa->settings.count;
                *pos++ = WLAN_EID_CHAN_SWITCH_PARAM;
                *pos++ = 6;
                if (ifmsh->csa_role == IEEE80211_MESH_CSA_ROLE_INIT) {
                        *pos++ = ifmsh->mshcfg.dot11MeshTTL;
                        *pos |= WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR;
                } else {
                        *pos++ = ifmsh->chsw_ttl;
                }
                *pos++ |= csa->settings.block_tx ?
                          WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT : 0x00;
                put_unaligned_le16(WLAN_REASON_MESH_CHAN, pos);
                pos += 2;
                put_unaligned_le16(ifmsh->pre_value, pos);
                pos += 2;

                switch (csa->settings.chandef.width) {
                case NL80211_CHAN_WIDTH_40:
                        ie_len = 2 + sizeof(struct ieee80211_sec_chan_offs_ie);
                        pos = skb_put_zero(skb, ie_len);

                        *pos++ = WLAN_EID_SECONDARY_CHANNEL_OFFSET; /* EID */
                        *pos++ = 1;                                    /* len */
                        ct = cfg80211_get_chandef_type(&csa->settings.chandef);
                        if (ct == NL80211_CHAN_HT40PLUS)
                                *pos++ = IEEE80211_HT_PARAM_CHA_SEC_ABOVE;
                        else
                                *pos++ = IEEE80211_HT_PARAM_CHA_SEC_BELOW;
                        break;
                case NL80211_CHAN_WIDTH_80:
                case NL80211_CHAN_WIDTH_80P80:
                case NL80211_CHAN_WIDTH_160:
                        /* Channel Switch Wrapper + Wide Bandwidth CSA IE */
                        ie_len = 2 + 2 +
                                 sizeof(struct ieee80211_wide_bw_chansw_ie);
                        pos = skb_put_zero(skb, ie_len);

                        *pos++ = WLAN_EID_CHANNEL_SWITCH_WRAPPER; /* EID */
                        *pos++ = 5;                                  /* len */
                        /* put sub IE */
                        chandef = &csa->settings.chandef;
                        ieee80211_ie_build_wide_bw_cs(pos, chandef);
                        break;
                default:
                        break;
                }
        }
        rcu_read_unlock();

        if (ieee80211_put_srates_elem(skb, sband,
                                      sdata->vif.bss_conf.basic_rates,
                                      0, WLAN_EID_SUPP_RATES) ||
            mesh_add_ds_params_ie(sdata, skb))
                goto out_free;

        bcn->head_len = skb->len;
        memcpy(bcn->head, skb->data, bcn->head_len);

        /* now the tail */
        skb_trim(skb, 0);
        bcn->tail = bcn->head + bcn->head_len;

        if (ieee80211_put_srates_elem(skb, sband,
                                      sdata->vif.bss_conf.basic_rates,
                                      0, WLAN_EID_EXT_SUPP_RATES) ||
            mesh_add_rsn_ie(sdata, skb) ||
            mesh_add_ht_cap_ie(sdata, skb) ||
            mesh_add_ht_oper_ie(sdata, skb) ||
            mesh_add_meshid_ie(sdata, skb) ||
            mesh_add_meshconf_ie(sdata, skb) ||
            mesh_add_awake_window_ie(sdata, skb) ||
            mesh_add_vht_cap_ie(sdata, skb) ||
            mesh_add_vht_oper_ie(sdata, skb) ||
            mesh_add_he_cap_ie(sdata, skb, ie_len_he_cap) ||
            mesh_add_he_oper_ie(sdata, skb) ||
            mesh_add_he_6ghz_cap_ie(sdata, skb) ||
            mesh_add_eht_cap_ie(sdata, skb, ie_len_eht_cap) ||
            mesh_add_eht_oper_ie(sdata, skb) ||
            mesh_add_vendor_ies(sdata, skb))
                goto out_free;

        bcn->tail_len = skb->len;
        memcpy(bcn->tail, skb->data, bcn->tail_len);
        ieee80211_mesh_update_bss_params(sdata, bcn->tail, bcn->tail_len);
        bcn->meshconf = (struct ieee80211_meshconf_ie *)
                                        (bcn->tail + ifmsh->meshconf_offset);

        dev_kfree_skb(skb);
        rcu_assign_pointer(ifmsh->beacon, bcn);
        return 0;
out_free:
        kfree(bcn);
        dev_kfree_skb(skb);
        return -ENOMEM;
}

static int
ieee80211_mesh_rebuild_beacon(struct ieee80211_sub_if_data *sdata)
{
        struct beacon_data *old_bcn;
        int ret;

        old_bcn = sdata_dereference(sdata->u.mesh.beacon, sdata);
        ret = ieee80211_mesh_build_beacon(&sdata->u.mesh);
        if (ret)
                /* just reuse old beacon */
                return ret;

        if (old_bcn)
                kfree_rcu(old_bcn, rcu_head);
        return 0;
}

void ieee80211_mbss_info_change_notify(struct ieee80211_sub_if_data *sdata,
                                       u64 changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        unsigned long bits[] = { BITMAP_FROM_U64(changed) };
        u32 bit;

        if (!changed)
                return;

        /* if we race with running work, worst case this work becomes a noop */
        for_each_set_bit(bit, bits, sizeof(changed) * BITS_PER_BYTE)
                set_bit(bit, ifmsh->mbss_changed);
        set_bit(MESH_WORK_MBSS_CHANGED, &ifmsh->wrkq_flags);
        wiphy_work_queue(sdata->local->hw.wiphy, &sdata->work);
}

int ieee80211_start_mesh(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_local *local = sdata->local;
        u64 changed = BSS_CHANGED_BEACON |
                      BSS_CHANGED_BEACON_ENABLED |
                      BSS_CHANGED_HT |
                      BSS_CHANGED_BASIC_RATES |
                      BSS_CHANGED_BEACON_INT |
                      BSS_CHANGED_MCAST_RATE;

        local->fif_other_bss++;
        /* mesh ifaces must set allmulti to forward mcast traffic */
        atomic_inc(&local->iff_allmultis);
        ieee80211_configure_filter(local);

        ifmsh->mesh_cc_id = 0;        /* Disabled */
        /* register sync ops from extensible synchronization framework */
        ifmsh->sync_ops = ieee80211_mesh_sync_ops_get(ifmsh->mesh_sp_id);
        ifmsh->sync_offset_clockdrift_max = 0;
        set_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags);
        ieee80211_mesh_root_setup(ifmsh);
        wiphy_work_queue(local->hw.wiphy, &sdata->work);
        sdata->vif.bss_conf.ht_operation_mode =
                                ifmsh->mshcfg.ht_opmode;
        sdata->vif.bss_conf.enable_beacon = true;

        changed |= ieee80211_mps_local_status_update(sdata);

        if (ieee80211_mesh_build_beacon(ifmsh)) {
                ieee80211_stop_mesh(sdata);
                return -ENOMEM;
        }

        ieee80211_recalc_dtim(local, sdata);
        ieee80211_link_info_change_notify(sdata, &sdata->deflink, changed);

        netif_carrier_on(sdata->dev);
        return 0;
}

void ieee80211_stop_mesh(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct beacon_data *bcn;

        netif_carrier_off(sdata->dev);

        /* flush STAs and mpaths on this iface */
        sta_info_flush(sdata, -1);
        ieee80211_free_keys(sdata, true);
        mesh_path_flush_by_iface(sdata);

        /* stop the beacon */
        ifmsh->mesh_id_len = 0;
        sdata->vif.bss_conf.enable_beacon = false;
        sdata->beacon_rate_set = false;
        clear_bit(SDATA_STATE_OFFCHANNEL_BEACON_STOPPED, &sdata->state);
        ieee80211_link_info_change_notify(sdata, &sdata->deflink,
                                          BSS_CHANGED_BEACON_ENABLED);

        /* remove beacon */
        bcn = sdata_dereference(ifmsh->beacon, sdata);
        RCU_INIT_POINTER(ifmsh->beacon, NULL);
        kfree_rcu(bcn, rcu_head);

        /* free all potentially still buffered group-addressed frames */
        local->total_ps_buffered -= skb_queue_len(&ifmsh->ps.bc_buf);
        skb_queue_purge(&ifmsh->ps.bc_buf);

        timer_delete_sync(&sdata->u.mesh.housekeeping_timer);
        timer_delete_sync(&sdata->u.mesh.mesh_path_root_timer);
        timer_delete_sync(&sdata->u.mesh.mesh_path_timer);

        /* clear any mesh work (for next join) we may have accrued */
        ifmsh->wrkq_flags = 0;
        memset(ifmsh->mbss_changed, 0, sizeof(ifmsh->mbss_changed));

        local->fif_other_bss--;
        atomic_dec(&local->iff_allmultis);
        ieee80211_configure_filter(local);
}

static void ieee80211_mesh_csa_mark_radar(struct ieee80211_sub_if_data *sdata)
{
        int err;

        /* if the current channel is a DFS channel, mark the channel as
         * unavailable.
         */
        err = cfg80211_chandef_dfs_required(sdata->local->hw.wiphy,
                                            &sdata->vif.bss_conf.chanreq.oper,
                                            NL80211_IFTYPE_MESH_POINT);
        if (err > 0)
                cfg80211_radar_event(sdata->local->hw.wiphy,
                                     &sdata->vif.bss_conf.chanreq.oper,
                                     GFP_ATOMIC);
}

static bool
ieee80211_mesh_process_chnswitch(struct ieee80211_sub_if_data *sdata,
                                 struct ieee802_11_elems *elems, bool beacon)
{
        struct cfg80211_csa_settings params;
        struct ieee80211_csa_ie csa_ie;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee80211_supported_band *sband;
        int err;
        struct ieee80211_conn_settings conn = ieee80211_conn_settings_unlimited;
        u32 vht_cap_info = 0;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        sband = ieee80211_get_sband(sdata);
        if (!sband)
                return false;

        switch (sdata->vif.bss_conf.chanreq.oper.width) {
        case NL80211_CHAN_WIDTH_20_NOHT:
                conn.mode = IEEE80211_CONN_MODE_LEGACY;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        case NL80211_CHAN_WIDTH_20:
                conn.mode = IEEE80211_CONN_MODE_HT;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_20;
                break;
        case NL80211_CHAN_WIDTH_40:
                conn.mode = IEEE80211_CONN_MODE_HT;
                conn.bw_limit = IEEE80211_CONN_BW_LIMIT_40;
                break;
        default:
                break;
        }

        if (elems->vht_cap_elem)
                vht_cap_info =
                        le32_to_cpu(elems->vht_cap_elem->vht_cap_info);

        memset(&params, 0, sizeof(params));
        err = ieee80211_parse_ch_switch_ie(sdata, elems, sband->band,
                                           vht_cap_info, &conn,
                                           sdata->vif.addr, false,
                                           &csa_ie);
        if (err < 0)
                return false;
        if (err)
                return false;

        /* Mark the channel unavailable if the reason for the switch is
         * regulatory.
         */
        if (csa_ie.reason_code == WLAN_REASON_MESH_CHAN_REGULATORY)
                ieee80211_mesh_csa_mark_radar(sdata);

        params.chandef = csa_ie.chanreq.oper;
        params.count = csa_ie.count;

        if (!cfg80211_chandef_usable(sdata->local->hw.wiphy, &params.chandef,
                                     IEEE80211_CHAN_DISABLED) ||
            !cfg80211_reg_can_beacon(sdata->local->hw.wiphy, &params.chandef,
                                     NL80211_IFTYPE_MESH_POINT)) {
                sdata_info(sdata,
                           "mesh STA %pM switches to unsupported channel (%d MHz, width:%d, CF1/2: %d/%d MHz), aborting\n",
                           sdata->vif.addr,
                           params.chandef.chan->center_freq,
                           params.chandef.width,
                           params.chandef.center_freq1,
                           params.chandef.center_freq2);
                return false;
        }

        err = cfg80211_chandef_dfs_required(sdata->local->hw.wiphy,
                                            &params.chandef,
                                            NL80211_IFTYPE_MESH_POINT);
        if (err < 0)
                return false;
        if (err > 0 && !ifmsh->userspace_handles_dfs) {
                sdata_info(sdata,
                           "mesh STA %pM switches to channel requiring DFS (%d MHz, width:%d, CF1/2: %d/%d MHz), aborting\n",
                           sdata->vif.addr,
                           params.chandef.chan->center_freq,
                           params.chandef.width,
                           params.chandef.center_freq1,
                           params.chandef.center_freq2);
                return false;
        }

        params.radar_required = err;

        if (cfg80211_chandef_identical(&params.chandef,
                                       &sdata->vif.bss_conf.chanreq.oper)) {
                mcsa_dbg(sdata,
                         "received csa with an identical chandef, ignoring\n");
                return true;
        }

        mcsa_dbg(sdata,
                 "received channel switch announcement to go to channel %d MHz\n",
                 params.chandef.chan->center_freq);

        params.block_tx = csa_ie.mode & WLAN_EID_CHAN_SWITCH_PARAM_TX_RESTRICT;
        if (beacon) {
                ifmsh->chsw_ttl = csa_ie.ttl - 1;
                if (ifmsh->pre_value >= csa_ie.pre_value)
                        return false;
                ifmsh->pre_value = csa_ie.pre_value;
        }

        if (ifmsh->chsw_ttl >= ifmsh->mshcfg.dot11MeshTTL)
                return false;

        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_REPEATER;

        if (ieee80211_channel_switch(sdata->local->hw.wiphy, sdata->dev,
                                     &params) < 0)
                return false;

        return true;
}

static void
ieee80211_mesh_rx_probe_req(struct ieee80211_sub_if_data *sdata,
                            struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct sk_buff *presp;
        struct beacon_data *bcn;
        struct ieee80211_mgmt *hdr;
        struct ieee802_11_elems *elems;
        size_t baselen;
        u8 *pos;

        pos = mgmt->u.probe_req.variable;
        baselen = (u8 *) pos - (u8 *) mgmt;
        if (baselen > len)
                return;

        elems = ieee802_11_parse_elems(pos, len - baselen, false, NULL);
        if (!elems)
                return;

        if (!elems->mesh_id)
                goto free;

        /* 802.11-2012 10.1.4.3.2 */
        if ((!ether_addr_equal(mgmt->da, sdata->vif.addr) &&
             !is_broadcast_ether_addr(mgmt->da)) ||
            elems->ssid_len != 0)
                goto free;

        if (elems->mesh_id_len != 0 &&
            (elems->mesh_id_len != ifmsh->mesh_id_len ||
             memcmp(elems->mesh_id, ifmsh->mesh_id, ifmsh->mesh_id_len)))
                goto free;

        rcu_read_lock();
        bcn = rcu_dereference(ifmsh->beacon);

        if (!bcn)
                goto out;

        presp = dev_alloc_skb(local->tx_headroom +
                              bcn->head_len + bcn->tail_len);
        if (!presp)
                goto out;

        skb_reserve(presp, local->tx_headroom);
        skb_put_data(presp, bcn->head, bcn->head_len);
        skb_put_data(presp, bcn->tail, bcn->tail_len);
        hdr = (struct ieee80211_mgmt *) presp->data;
        hdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
                                         IEEE80211_STYPE_PROBE_RESP);
        memcpy(hdr->da, mgmt->sa, ETH_ALEN);
        IEEE80211_SKB_CB(presp)->flags |= IEEE80211_TX_INTFL_DONT_ENCRYPT;
        ieee80211_tx_skb(sdata, presp);
out:
        rcu_read_unlock();
free:
        kfree(elems);
}

static void ieee80211_mesh_rx_bcn_presp(struct ieee80211_sub_if_data *sdata,
                                        u16 stype,
                                        struct ieee80211_mgmt *mgmt,
                                        size_t len,
                                        struct ieee80211_rx_status *rx_status)
{
        struct ieee80211_local *local = sdata->local;
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee802_11_elems *elems;
        struct ieee80211_channel *channel;
        size_t baselen;
        int freq;
        enum nl80211_band band = rx_status->band;

        /* ignore ProbeResp to foreign address */
        if (stype == IEEE80211_STYPE_PROBE_RESP &&
            !ether_addr_equal(mgmt->da, sdata->vif.addr))
                return;

        baselen = (u8 *) mgmt->u.probe_resp.variable - (u8 *) mgmt;
        if (baselen > len)
                return;

        elems = ieee802_11_parse_elems(mgmt->u.probe_resp.variable,
                                       len - baselen,
                                       false, NULL);
        if (!elems)
                return;

        /* ignore non-mesh or secure / insecure mismatch */
        if ((!elems->mesh_id || !elems->mesh_config) ||
            (elems->rsn && sdata->u.mesh.security == IEEE80211_MESH_SEC_NONE) ||
            (!elems->rsn && sdata->u.mesh.security != IEEE80211_MESH_SEC_NONE))
                goto free;

        if (elems->ds_params)
                freq = ieee80211_channel_to_frequency(elems->ds_params[0], band);
        else
                freq = rx_status->freq;

        channel = ieee80211_get_channel(local->hw.wiphy, freq);

        if (!channel || channel->flags & IEEE80211_CHAN_DISABLED)
                goto free;

        if (mesh_matches_local(sdata, elems)) {
                mpl_dbg(sdata, "rssi_threshold=%d,rx_status->signal=%d\n",
                        sdata->u.mesh.mshcfg.rssi_threshold, rx_status->signal);
                if (!sdata->u.mesh.user_mpm ||
                    sdata->u.mesh.mshcfg.rssi_threshold == 0 ||
                    sdata->u.mesh.mshcfg.rssi_threshold < rx_status->signal)
                        mesh_neighbour_update(sdata, mgmt->sa, elems,
                                              rx_status);

                if (ifmsh->csa_role != IEEE80211_MESH_CSA_ROLE_INIT &&
                    !sdata->vif.bss_conf.csa_active)
                        ieee80211_mesh_process_chnswitch(sdata, elems, true);
        }

        if (ifmsh->sync_ops)
                ifmsh->sync_ops->rx_bcn_presp(sdata, stype, mgmt, len,
                                              elems->mesh_config, rx_status);
free:
        kfree(elems);
}

int ieee80211_mesh_finish_csa(struct ieee80211_sub_if_data *sdata, u64 *changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct mesh_csa_settings *tmp_csa_settings;
        int ret = 0;

        /* Reset the TTL value and Initiator flag */
        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE;
        ifmsh->chsw_ttl = 0;

        /* Remove the CSA and MCSP elements from the beacon */
        tmp_csa_settings = sdata_dereference(ifmsh->csa, sdata);
        RCU_INIT_POINTER(ifmsh->csa, NULL);
        if (tmp_csa_settings)
                kfree_rcu(tmp_csa_settings, rcu_head);
        ret = ieee80211_mesh_rebuild_beacon(sdata);
        if (ret)
                return -EINVAL;

        *changed |= BSS_CHANGED_BEACON;

        mcsa_dbg(sdata, "complete switching to center freq %d MHz",
                 sdata->vif.bss_conf.chanreq.oper.chan->center_freq);
        return 0;
}

int ieee80211_mesh_csa_beacon(struct ieee80211_sub_if_data *sdata,
                              struct cfg80211_csa_settings *csa_settings,
                              u64 *changed)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct mesh_csa_settings *tmp_csa_settings;
        int ret = 0;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        tmp_csa_settings = kmalloc(sizeof(*tmp_csa_settings),
                                   GFP_ATOMIC);
        if (!tmp_csa_settings)
                return -ENOMEM;

        memcpy(&tmp_csa_settings->settings, csa_settings,
               sizeof(struct cfg80211_csa_settings));

        rcu_assign_pointer(ifmsh->csa, tmp_csa_settings);

        ret = ieee80211_mesh_rebuild_beacon(sdata);
        if (ret) {
                tmp_csa_settings = rcu_dereference(ifmsh->csa);
                RCU_INIT_POINTER(ifmsh->csa, NULL);
                kfree_rcu(tmp_csa_settings, rcu_head);
                return ret;
        }

        *changed |= BSS_CHANGED_BEACON;
        return 0;
}

static int mesh_fwd_csa_frame(struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_mgmt *mgmt, size_t len,
                               struct ieee802_11_elems *elems)
{
        struct ieee80211_mgmt *mgmt_fwd;
        struct sk_buff *skb;
        struct ieee80211_local *local = sdata->local;

        skb = dev_alloc_skb(local->tx_headroom + len);
        if (!skb)
                return -ENOMEM;
        skb_reserve(skb, local->tx_headroom);
        mgmt_fwd = skb_put(skb, len);

        elems->mesh_chansw_params_ie->mesh_ttl--;
        elems->mesh_chansw_params_ie->mesh_flags &=
                ~WLAN_EID_CHAN_SWITCH_PARAM_INITIATOR;

        memcpy(mgmt_fwd, mgmt, len);
        eth_broadcast_addr(mgmt_fwd->da);
        memcpy(mgmt_fwd->sa, sdata->vif.addr, ETH_ALEN);
        memcpy(mgmt_fwd->bssid, sdata->vif.addr, ETH_ALEN);

        ieee80211_tx_skb(sdata, skb);
        return 0;
}

static void mesh_rx_csa_frame(struct ieee80211_sub_if_data *sdata,
                              struct ieee80211_mgmt *mgmt, size_t len)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        struct ieee802_11_elems *elems;
        u16 pre_value;
        bool fwd_csa = true;
        size_t baselen;
        u8 *pos;

        if (mgmt->u.action.u.measurement.action_code !=
            WLAN_ACTION_SPCT_CHL_SWITCH)
                return;

        pos = mgmt->u.action.u.chan_switch.variable;
        baselen = offsetof(struct ieee80211_mgmt,
                           u.action.u.chan_switch.variable);
        elems = ieee802_11_parse_elems(pos, len - baselen, true, NULL);
        if (!elems)
                return;

        if (!mesh_matches_local(sdata, elems))
                goto free;

        ifmsh->chsw_ttl = elems->mesh_chansw_params_ie->mesh_ttl;
        if (!--ifmsh->chsw_ttl)
                fwd_csa = false;

        pre_value = le16_to_cpu(elems->mesh_chansw_params_ie->mesh_pre_value);
        if (ifmsh->pre_value >= pre_value)
                goto free;

        ifmsh->pre_value = pre_value;

        if (!sdata->vif.bss_conf.csa_active &&
            !ieee80211_mesh_process_chnswitch(sdata, elems, false)) {
                mcsa_dbg(sdata, "Failed to process CSA action frame");
                goto free;
        }

        /* forward or re-broadcast the CSA frame */
        if (fwd_csa) {
                if (mesh_fwd_csa_frame(sdata, mgmt, len, elems) < 0)
                        mcsa_dbg(sdata, "Failed to forward the CSA frame");
        }
free:
        kfree(elems);
}

static void ieee80211_mesh_rx_mgmt_action(struct ieee80211_sub_if_data *sdata,
                                          struct ieee80211_mgmt *mgmt,
                                          size_t len,
                                          struct ieee80211_rx_status *rx_status)
{
        switch (mgmt->u.action.category) {
        case WLAN_CATEGORY_SELF_PROTECTED:
                switch (mgmt->u.action.u.self_prot.action_code) {
                case WLAN_SP_MESH_PEERING_OPEN:
                case WLAN_SP_MESH_PEERING_CLOSE:
                case WLAN_SP_MESH_PEERING_CONFIRM:
                        mesh_rx_plink_frame(sdata, mgmt, len, rx_status);
                        break;
                }
                break;
        case WLAN_CATEGORY_MESH_ACTION:
                if (mesh_action_is_path_sel(mgmt))
                        mesh_rx_path_sel_frame(sdata, mgmt, len);
                break;
        case WLAN_CATEGORY_SPECTRUM_MGMT:
                mesh_rx_csa_frame(sdata, mgmt, len);
                break;
        }
}

void ieee80211_mesh_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
                                   struct sk_buff *skb)
{
        struct ieee80211_rx_status *rx_status;
        struct ieee80211_mgmt *mgmt;
        u16 stype;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        /* mesh already went down */
        if (!sdata->u.mesh.mesh_id_len)
                return;

        rx_status = IEEE80211_SKB_RXCB(skb);
        mgmt = (struct ieee80211_mgmt *) skb->data;
        stype = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_STYPE;

        switch (stype) {
        case IEEE80211_STYPE_PROBE_RESP:
        case IEEE80211_STYPE_BEACON:
                ieee80211_mesh_rx_bcn_presp(sdata, stype, mgmt, skb->len,
                                            rx_status);
                break;
        case IEEE80211_STYPE_PROBE_REQ:
                ieee80211_mesh_rx_probe_req(sdata, mgmt, skb->len);
                break;
        case IEEE80211_STYPE_ACTION:
                ieee80211_mesh_rx_mgmt_action(sdata, mgmt, skb->len, rx_status);
                break;
        }
}

static void mesh_bss_info_changed(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        u32 bit;
        u64 changed = 0;

        for_each_set_bit(bit, ifmsh->mbss_changed,
                         sizeof(changed) * BITS_PER_BYTE) {
                clear_bit(bit, ifmsh->mbss_changed);
                changed |= BIT(bit);
        }

        if (sdata->vif.bss_conf.enable_beacon &&
            (changed & (BSS_CHANGED_BEACON |
                        BSS_CHANGED_HT |
                        BSS_CHANGED_BASIC_RATES |
                        BSS_CHANGED_BEACON_INT)))
                if (ieee80211_mesh_rebuild_beacon(sdata))
                        return;

        ieee80211_link_info_change_notify(sdata, &sdata->deflink, changed);
}

void ieee80211_mesh_work(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;

        lockdep_assert_wiphy(sdata->local->hw.wiphy);

        /* mesh already went down */
        if (!sdata->u.mesh.mesh_id_len)
                return;

        if (ifmsh->preq_queue_len &&
            time_after(jiffies,
                       ifmsh->last_preq + msecs_to_jiffies(ifmsh->mshcfg.dot11MeshHWMPpreqMinInterval)))
                mesh_path_start_discovery(sdata);

        if (test_and_clear_bit(MESH_WORK_HOUSEKEEPING, &ifmsh->wrkq_flags))
                ieee80211_mesh_housekeeping(sdata);

        if (test_and_clear_bit(MESH_WORK_ROOT, &ifmsh->wrkq_flags))
                ieee80211_mesh_rootpath(sdata);

        if (test_and_clear_bit(MESH_WORK_DRIFT_ADJUST, &ifmsh->wrkq_flags))
                mesh_sync_adjust_tsf(sdata);

        if (test_and_clear_bit(MESH_WORK_MBSS_CHANGED, &ifmsh->wrkq_flags))
                mesh_bss_info_changed(sdata);
}


void ieee80211_mesh_init_sdata(struct ieee80211_sub_if_data *sdata)
{
        struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
        static u8 zero_addr[ETH_ALEN] = {};

        timer_setup(&ifmsh->housekeeping_timer,
                    ieee80211_mesh_housekeeping_timer, 0);

        ifmsh->accepting_plinks = true;
        atomic_set(&ifmsh->mpaths, 0);
        mesh_rmc_init(sdata);
        ifmsh->last_preq = jiffies;
        ifmsh->next_perr = jiffies;
        ifmsh->csa_role = IEEE80211_MESH_CSA_ROLE_NONE;
        ifmsh->nonpeer_pm = NL80211_MESH_POWER_ACTIVE;
        /* Allocate all mesh structures when creating the first mesh interface. */
        if (!mesh_allocated)
                ieee80211s_init();

        mesh_pathtbl_init(sdata);

        timer_setup(&ifmsh->mesh_path_timer, ieee80211_mesh_path_timer, 0);
        timer_setup(&ifmsh->mesh_path_root_timer,
                    ieee80211_mesh_path_root_timer, 0);
        INIT_LIST_HEAD(&ifmsh->preq_queue.list);
        skb_queue_head_init(&ifmsh->ps.bc_buf);
        spin_lock_init(&ifmsh->mesh_preq_queue_lock);
        spin_lock_init(&ifmsh->sync_offset_lock);
        RCU_INIT_POINTER(ifmsh->beacon, NULL);

        sdata->vif.bss_conf.bssid = zero_addr;
}

void ieee80211_mesh_teardown_sdata(struct ieee80211_sub_if_data *sdata)
{
        mesh_rmc_free(sdata);
        mesh_pathtbl_unregister(sdata);
}
















































































  492 
   11 









  500 


  494 
    1 



  500 



  495 









  531 







   14 


  501 




































    1 
   89 
  133 














 1257 
 1263 
  939 
  774 






















   16 











  315 
    4 
































    6 


    6 



























    4 




    4 


    3 
    1 


















    4 
    4 






















    4 







   15 




















    1 



    1 

    1 





    1 





    1 
    1 






  191 






  192 








  427 











  426 
  429 


  429 
  340 


   87 
    6 


























    5 



















  666 






   17 



   17 

    2 
    2 


































    6 


    6 
    6 








    6 



























  470 
    2 
  471 





  447 

  448 

  471 










  108 

  361 






    2 










  489 
  301 


  474 









  488 




  490 





  206 









  299 


  468 












    4 


  364 
  108 
















  455 





   12 

   11 
  320 










    1 



  318 


  320 

  320 















  387 
  513 










  151 
  412 









    1 




   27 



   25 


   26 



    2 
    1 
    1 
    1 












   11 








   14 

   11 
    1 
    1 










   85 



    5 


   82 






   45 
   14 







    1 















































  530 





    6 





   38 





   19 








    6 

    6 

    6 














    1 




    6 





    1 





  531 












  398 





    2 


    2 


  345 



  396 








  398 

   16 










    4 
















  387 
    8 
    8 
    2 



    7 









    5 
    5 

    5 









  382 





  358 




  504 





  506 


  127 


  382 






  358 






   33 












  356 



    5 


  376 








    6 

    6 




   13 







  365 


    3 
    2 

















  383 
  382 


  378 

    4 




    1 

    2 












  289 






















  503 



   51 

   76 



  385 



   94 
   95 
   95 



   95 





   95 




















































   32 




  141 


























  318 








  318 
  319 










  318 




  319 





































  342 


  345 

  342 















  344 



  343 






   29 


   28 
   29 

   28 


   29 









   51 


   50 

   50 

   16 
   32 














    7 
















   32 




   33 

    2 
   32 
   31 

   32 














   29 

   29 

   28 









































    2 





    2 



















































































    1 







   76 








   75 
   75 




   38 
    7 












































    7 



    7 
















  277 














































































































































































    7 






















































   44 


    1 

   45 





   15 



   15 
   15 
   15 
   14 















    1 


    5 
    1 





   27 






   27 

    5 
   21 

















































    1 




    9 
    9 











    1 





    2 














   13 

    1 






   26 


    1 







    1 
    9 










    5 









    5 






    4 


    1 












    1 











































































































































    1 









    1 









    1 

    1 
    1 






    1 




    1 












    1 






























































































































































































































    1 


    1 






























    1 



    1 

    1 









    1 


















    1 











    1 
    1 





































































































































































 1052 





  812 
  424 

  273 










  769 
































    1 












   16 













  769 



















    1 











  769 
  320 


  468 


  756 






  757 




   15 





  738 
    5 







    1 












    1 














    1 





























































































  336 
  410 

  731 
    6 
  740 




  740 

    7 

  741 








  211 




  103 


  128 



   22 




   96 
  127 

  129 
































































    9 





    4 





  541 

















  406 





  863 






  351 



  534 












   95 




   92 
   96 

    5 

   99 


    3 


   92 







    1 












  338 


  301 

    7 

    4 


   30 


































   17 






    1 




    1 
    1 



   14 




    8 
    6 
















































    7 



    1 




    6 






































    3 


   33 
   17 


   35 


   32 







   10 


    8 



    1 


    1 


    1 


    2 









    8 





    3 
    2 











  670 

    2 

  671 







   87 









    2 

   28 
   29 


   11 






    6 


   63 



   23 
    1 

   24 








































































































































































































   13 






    8 
    2 

    3 























    8 



    8 



    3 














   17 










    2 


    1 


    8 
    6 





   10 
    3 









































































































   35 

































   27 






















   19 







   15 


   14 








   27 


   17 





    2 


    8 





   19 
    9 


    4 




    5 


    8 


    4 
    7 


















   38 






    1 



    1 













    8 



   29 

    1 

   27 



   28 





   30 







   37 



   36 

   37 





    2 






   33 



   37 























    8 


    4 












   25 


    3 










   32 













   44 



   44 


   32 



















    5 





    9 






   25 



   25 


   13 












































   17 




   20 









   15 
    1 












    3 
   15 
   15 
    3 



    3 
















   14 



    2 

    3 
   10 



























   21 



   12 



    9 



    6 





  419 



    4 







    5 
  407 




    5 



    8 














   22 



   21 




   21 





  408 



  409 



































































































































































   23 








    2 


   20 




    2 

   18 














































































































































































    9 
    3 
    9 








    7 



    6 
    7 












   12 




    4 


    1 

    7 



































   27 


























































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/signal.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  1997-11-02  Modified for POSIX.1b signals by Richard Henderson
 *
 *  2003-06-02  Jim Houston - Concurrent Computer Corp.
 *                Changes to use preallocated sigqueue structures
 *                to allow signals to be sent reliably.
 */

#include <linux/slab.h>
#include <linux/export.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/user.h>
#include <linux/sched/debug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/proc_fs.h>
#include <linux/tty.h>
#include <linux/binfmts.h>
#include <linux/coredump.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/signalfd.h>
#include <linux/ratelimit.h>
#include <linux/task_work.h>
#include <linux/capability.h>
#include <linux/freezer.h>
#include <linux/pid_namespace.h>
#include <linux/nsproxy.h>
#include <linux/user_namespace.h>
#include <linux/uprobes.h>
#include <linux/compat.h>
#include <linux/cn_proc.h>
#include <linux/compiler.h>
#include <linux/posix-timers.h>
#include <linux/cgroup.h>
#include <linux/audit.h>
#include <linux/sysctl.h>
#include <uapi/linux/pidfd.h>

#define CREATE_TRACE_POINTS
#include <trace/events/signal.h>

#include <asm/param.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
#include <asm/siginfo.h>
#include <asm/cacheflush.h>
#include <asm/syscall.h>        /* for syscall_get_* */

#include "time/posix-timers.h"

/*
 * SLAB caches for signal bits.
 */

static struct kmem_cache *sigqueue_cachep;

int print_fatal_signals __read_mostly;

static void __user *sig_handler(struct task_struct *t, int sig)
{
        return t->sighand->action[sig - 1].sa.sa_handler;
}

static inline bool sig_handler_ignored(void __user *handler, int sig)
{
        /* Is it explicitly or implicitly ignored? */
        return handler == SIG_IGN ||
               (handler == SIG_DFL && sig_kernel_ignore(sig));
}

static bool sig_task_ignored(struct task_struct *t, int sig, bool force)
{
        void __user *handler;

        handler = sig_handler(t, sig);

        /* SIGKILL and SIGSTOP may not be sent to the global init */
        if (unlikely(is_global_init(t) && sig_kernel_only(sig)))
                return true;

        if (unlikely(t->signal->flags & SIGNAL_UNKILLABLE) &&
            handler == SIG_DFL && !(force && sig_kernel_only(sig)))
                return true;

        /* Only allow kernel generated signals to this kthread */
        if (unlikely((t->flags & PF_KTHREAD) &&
                     (handler == SIG_KTHREAD_KERNEL) && !force))
                return true;

        return sig_handler_ignored(handler, sig);
}

static bool sig_ignored(struct task_struct *t, int sig, bool force)
{
        /*
         * Blocked signals are never ignored, since the
         * signal handler may change by the time it is
         * unblocked.
         */
        if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig))
                return false;

        /*
         * Tracers may want to know about even ignored signal unless it
         * is SIGKILL which can't be reported anyway but can be ignored
         * by SIGNAL_UNKILLABLE task.
         */
        if (t->ptrace && sig != SIGKILL)
                return false;

        return sig_task_ignored(t, sig, force);
}

/*
 * Re-calculate pending state from the set of locally pending
 * signals, globally pending signals, and blocked signals.
 */
static inline bool has_pending_signals(sigset_t *signal, sigset_t *blocked)
{
        unsigned long ready;
        long i;

        switch (_NSIG_WORDS) {
        default:
                for (i = _NSIG_WORDS, ready = 0; --i >= 0 ;)
                        ready |= signal->sig[i] &~ blocked->sig[i];
                break;

        case 4: ready  = signal->sig[3] &~ blocked->sig[3];
                ready |= signal->sig[2] &~ blocked->sig[2];
                ready |= signal->sig[1] &~ blocked->sig[1];
                ready |= signal->sig[0] &~ blocked->sig[0];
                break;

        case 2: ready  = signal->sig[1] &~ blocked->sig[1];
                ready |= signal->sig[0] &~ blocked->sig[0];
                break;

        case 1: ready  = signal->sig[0] &~ blocked->sig[0];
        }
        return ready !=        0;
}

#define PENDING(p,b) has_pending_signals(&(p)->signal, (b))

static bool recalc_sigpending_tsk(struct task_struct *t)
{
        if ((t->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) ||
            PENDING(&t->pending, &t->blocked) ||
            PENDING(&t->signal->shared_pending, &t->blocked) ||
            cgroup_task_frozen(t)) {
                set_tsk_thread_flag(t, TIF_SIGPENDING);
                return true;
        }

        /*
         * We must never clear the flag in another thread, or in current
         * when it's possible the current syscall is returning -ERESTART*.
         * So we don't clear it here, and only callers who know they should do.
         */
        return false;
}

void recalc_sigpending(void)
{
        if (!recalc_sigpending_tsk(current) && !freezing(current)) {
                if (unlikely(test_thread_flag(TIF_SIGPENDING)))
                        clear_thread_flag(TIF_SIGPENDING);
        }
}
EXPORT_SYMBOL(recalc_sigpending);

void calculate_sigpending(void)
{
        /* Have any signals or users of TIF_SIGPENDING been delayed
         * until after fork?
         */
        spin_lock_irq(&current->sighand->siglock);
        set_tsk_thread_flag(current, TIF_SIGPENDING);
        recalc_sigpending();
        spin_unlock_irq(&current->sighand->siglock);
}

/* Given the mask, find the first available signal that should be serviced. */

#define SYNCHRONOUS_MASK \
        (sigmask(SIGSEGV) | sigmask(SIGBUS) | sigmask(SIGILL) | \
         sigmask(SIGTRAP) | sigmask(SIGFPE) | sigmask(SIGSYS))

int next_signal(struct sigpending *pending, sigset_t *mask)
{
        unsigned long i, *s, *m, x;
        int sig = 0;

        s = pending->signal.sig;
        m = mask->sig;

        /*
         * Handle the first word specially: it contains the
         * synchronous signals that need to be dequeued first.
         */
        x = *s &~ *m;
        if (x) {
                if (x & SYNCHRONOUS_MASK)
                        x &= SYNCHRONOUS_MASK;
                sig = ffz(~x) + 1;
                return sig;
        }

        switch (_NSIG_WORDS) {
        default:
                for (i = 1; i < _NSIG_WORDS; ++i) {
                        x = *++s &~ *++m;
                        if (!x)
                                continue;
                        sig = ffz(~x) + i*_NSIG_BPW + 1;
                        break;
                }
                break;

        case 2:
                x = s[1] &~ m[1];
                if (!x)
                        break;
                sig = ffz(~x) + _NSIG_BPW + 1;
                break;

        case 1:
                /* Nothing to do */
                break;
        }

        return sig;
}

static inline void print_dropped_signal(int sig)
{
        static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);

        if (!print_fatal_signals)
                return;

        if (!__ratelimit(&ratelimit_state))
                return;

        pr_info("%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n",
                                current->comm, current->pid, sig);
}

/**
 * task_set_jobctl_pending - set jobctl pending bits
 * @task: target task
 * @mask: pending bits to set
 *
 * Clear @mask from @task->jobctl.  @mask must be subset of
 * %JOBCTL_PENDING_MASK | %JOBCTL_STOP_CONSUME | %JOBCTL_STOP_SIGMASK |
 * %JOBCTL_TRAPPING.  If stop signo is being set, the existing signo is
 * cleared.  If @task is already being killed or exiting, this function
 * becomes noop.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 *
 * RETURNS:
 * %true if @mask is set, %false if made noop because @task was dying.
 */
bool task_set_jobctl_pending(struct task_struct *task, unsigned long mask)
{
        BUG_ON(mask & ~(JOBCTL_PENDING_MASK | JOBCTL_STOP_CONSUME |
                        JOBCTL_STOP_SIGMASK | JOBCTL_TRAPPING));
        BUG_ON((mask & JOBCTL_TRAPPING) && !(mask & JOBCTL_PENDING_MASK));

        if (unlikely(fatal_signal_pending(task) || (task->flags & PF_EXITING)))
                return false;

        if (mask & JOBCTL_STOP_SIGMASK)
                task->jobctl &= ~JOBCTL_STOP_SIGMASK;

        task->jobctl |= mask;
        return true;
}

/**
 * task_clear_jobctl_trapping - clear jobctl trapping bit
 * @task: target task
 *
 * If JOBCTL_TRAPPING is set, a ptracer is waiting for us to enter TRACED.
 * Clear it and wake up the ptracer.  Note that we don't need any further
 * locking.  @task->siglock guarantees that @task->parent points to the
 * ptracer.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
void task_clear_jobctl_trapping(struct task_struct *task)
{
        if (unlikely(task->jobctl & JOBCTL_TRAPPING)) {
                task->jobctl &= ~JOBCTL_TRAPPING;
                smp_mb();        /* advised by wake_up_bit() */
                wake_up_bit(&task->jobctl, JOBCTL_TRAPPING_BIT);
        }
}

/**
 * task_clear_jobctl_pending - clear jobctl pending bits
 * @task: target task
 * @mask: pending bits to clear
 *
 * Clear @mask from @task->jobctl.  @mask must be subset of
 * %JOBCTL_PENDING_MASK.  If %JOBCTL_STOP_PENDING is being cleared, other
 * STOP bits are cleared together.
 *
 * If clearing of @mask leaves no stop or trap pending, this function calls
 * task_clear_jobctl_trapping().
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
void task_clear_jobctl_pending(struct task_struct *task, unsigned long mask)
{
        BUG_ON(mask & ~JOBCTL_PENDING_MASK);

        if (mask & JOBCTL_STOP_PENDING)
                mask |= JOBCTL_STOP_CONSUME | JOBCTL_STOP_DEQUEUED;

        task->jobctl &= ~mask;

        if (!(task->jobctl & JOBCTL_PENDING_MASK))
                task_clear_jobctl_trapping(task);
}

/**
 * task_participate_group_stop - participate in a group stop
 * @task: task participating in a group stop
 *
 * @task has %JOBCTL_STOP_PENDING set and is participating in a group stop.
 * Group stop states are cleared and the group stop count is consumed if
 * %JOBCTL_STOP_CONSUME was set.  If the consumption completes the group
 * stop, the appropriate `SIGNAL_*` flags are set.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 *
 * RETURNS:
 * %true if group stop completion should be notified to the parent, %false
 * otherwise.
 */
static bool task_participate_group_stop(struct task_struct *task)
{
        struct signal_struct *sig = task->signal;
        bool consume = task->jobctl & JOBCTL_STOP_CONSUME;

        WARN_ON_ONCE(!(task->jobctl & JOBCTL_STOP_PENDING));

        task_clear_jobctl_pending(task, JOBCTL_STOP_PENDING);

        if (!consume)
                return false;

        if (!WARN_ON_ONCE(sig->group_stop_count == 0))
                sig->group_stop_count--;

        /*
         * Tell the caller to notify completion iff we are entering into a
         * fresh group stop.  Read comment in do_signal_stop() for details.
         */
        if (!sig->group_stop_count && !(sig->flags & SIGNAL_STOP_STOPPED)) {
                signal_set_stop_flags(sig, SIGNAL_STOP_STOPPED);
                return true;
        }
        return false;
}

void task_join_group_stop(struct task_struct *task)
{
        unsigned long mask = current->jobctl & JOBCTL_STOP_SIGMASK;
        struct signal_struct *sig = current->signal;

        if (sig->group_stop_count) {
                sig->group_stop_count++;
                mask |= JOBCTL_STOP_CONSUME;
        } else if (!(sig->flags & SIGNAL_STOP_STOPPED))
                return;

        /* Have the new thread join an on-going signal group stop */
        task_set_jobctl_pending(task, mask | JOBCTL_STOP_PENDING);
}

static struct ucounts *sig_get_ucounts(struct task_struct *t, int sig,
                                       int override_rlimit)
{
        struct ucounts *ucounts;
        long sigpending;

        /*
         * Protect access to @t credentials. This can go away when all
         * callers hold rcu read lock.
         *
         * NOTE! A pending signal will hold on to the user refcount,
         * and we get/put the refcount only when the sigpending count
         * changes from/to zero.
         */
        rcu_read_lock();
        ucounts = task_ucounts(t);
        sigpending = inc_rlimit_get_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING,
                                            override_rlimit);
        rcu_read_unlock();
        if (!sigpending)
                return NULL;

        if (unlikely(!override_rlimit && sigpending > task_rlimit(t, RLIMIT_SIGPENDING))) {
                dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
                print_dropped_signal(sig);
                return NULL;
        }

        return ucounts;
}

static void __sigqueue_init(struct sigqueue *q, struct ucounts *ucounts,
                            const unsigned int sigqueue_flags)
{
        INIT_LIST_HEAD(&q->list);
        q->flags = sigqueue_flags;
        q->ucounts = ucounts;
}

/*
 * allocate a new signal queue record
 * - this may be called without locks if and only if t == current, otherwise an
 *   appropriate lock must be held to stop the target task from exiting
 */
static struct sigqueue *sigqueue_alloc(int sig, struct task_struct *t, gfp_t gfp_flags,
                                       int override_rlimit)
{
        struct ucounts *ucounts = sig_get_ucounts(t, sig, override_rlimit);
        struct sigqueue *q;

        if (!ucounts)
                return NULL;

        q = kmem_cache_alloc(sigqueue_cachep, gfp_flags);
        if (!q) {
                dec_rlimit_put_ucounts(ucounts, UCOUNT_RLIMIT_SIGPENDING);
                return NULL;
        }

        __sigqueue_init(q, ucounts, 0);
        return q;
}

static void __sigqueue_free(struct sigqueue *q)
{
        if (q->flags & SIGQUEUE_PREALLOC) {
                posixtimer_sigqueue_putref(q);
                return;
        }
        if (q->ucounts) {
                dec_rlimit_put_ucounts(q->ucounts, UCOUNT_RLIMIT_SIGPENDING);
                q->ucounts = NULL;
        }
        kmem_cache_free(sigqueue_cachep, q);
}

void flush_sigqueue(struct sigpending *queue)
{
        struct sigqueue *q;

        sigemptyset(&queue->signal);
        while (!list_empty(&queue->list)) {
                q = list_entry(queue->list.next, struct sigqueue , list);
                list_del_init(&q->list);
                __sigqueue_free(q);
        }
}

/*
 * Flush all pending signals for this kthread.
 */
void flush_signals(struct task_struct *t)
{
        unsigned long flags;

        spin_lock_irqsave(&t->sighand->siglock, flags);
        clear_tsk_thread_flag(t, TIF_SIGPENDING);
        flush_sigqueue(&t->pending);
        flush_sigqueue(&t->signal->shared_pending);
        spin_unlock_irqrestore(&t->sighand->siglock, flags);
}
EXPORT_SYMBOL(flush_signals);

void ignore_signals(struct task_struct *t)
{
        int i;

        for (i = 0; i < _NSIG; ++i)
                t->sighand->action[i].sa.sa_handler = SIG_IGN;

        flush_signals(t);
}

/*
 * Flush all handlers for a task.
 */

void
flush_signal_handlers(struct task_struct *t, int force_default)
{
        int i;
        struct k_sigaction *ka = &t->sighand->action[0];
        for (i = _NSIG ; i != 0 ; i--) {
                if (force_default || ka->sa.sa_handler != SIG_IGN)
                        ka->sa.sa_handler = SIG_DFL;
                ka->sa.sa_flags = 0;
#ifdef __ARCH_HAS_SA_RESTORER
                ka->sa.sa_restorer = NULL;
#endif
                sigemptyset(&ka->sa.sa_mask);
                ka++;
        }
}

bool unhandled_signal(struct task_struct *tsk, int sig)
{
        void __user *handler = tsk->sighand->action[sig-1].sa.sa_handler;
        if (is_global_init(tsk))
                return true;

        if (handler != SIG_IGN && handler != SIG_DFL)
                return false;

        /* If dying, we handle all new signals by ignoring them */
        if (fatal_signal_pending(tsk))
                return false;

        /* if ptraced, let the tracer determine */
        return !tsk->ptrace;
}

static void collect_signal(int sig, struct sigpending *list, kernel_siginfo_t *info,
                           struct sigqueue **timer_sigq)
{
        struct sigqueue *q, *first = NULL;

        /*
         * Collect the siginfo appropriate to this signal.  Check if
         * there is another siginfo for the same signal.
        */
        list_for_each_entry(q, &list->list, list) {
                if (q->info.si_signo == sig) {
                        if (first)
                                goto still_pending;
                        first = q;
                }
        }

        sigdelset(&list->signal, sig);

        if (first) {
still_pending:
                list_del_init(&first->list);
                copy_siginfo(info, &first->info);

                /*
                 * posix-timer signals are preallocated and freed when the last
                 * reference count is dropped in posixtimer_deliver_signal() or
                 * immediately on timer deletion when the signal is not pending.
                 * Spare the extra round through __sigqueue_free() which is
                 * ignoring preallocated signals.
                 */
                if (unlikely((first->flags & SIGQUEUE_PREALLOC) && (info->si_code == SI_TIMER)))
                        *timer_sigq = first;
                else
                        __sigqueue_free(first);
        } else {
                /*
                 * Ok, it wasn't in the queue.  This must be
                 * a fast-pathed signal or we must have been
                 * out of queue space.  So zero out the info.
                 */
                clear_siginfo(info);
                info->si_signo = sig;
                info->si_errno = 0;
                info->si_code = SI_USER;
                info->si_pid = 0;
                info->si_uid = 0;
        }
}

static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
                            kernel_siginfo_t *info, struct sigqueue **timer_sigq)
{
        int sig = next_signal(pending, mask);

        if (sig)
                collect_signal(sig, pending, info, timer_sigq);
        return sig;
}

/*
 * Try to dequeue a signal. If a deliverable signal is found fill in the
 * caller provided siginfo and return the signal number. Otherwise return
 * 0.
 */
int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type)
{
        struct task_struct *tsk = current;
        struct sigqueue *timer_sigq;
        int signr;

        lockdep_assert_held(&tsk->sighand->siglock);

again:
        *type = PIDTYPE_PID;
        timer_sigq = NULL;
        signr = __dequeue_signal(&tsk->pending, mask, info, &timer_sigq);
        if (!signr) {
                *type = PIDTYPE_TGID;
                signr = __dequeue_signal(&tsk->signal->shared_pending,
                                         mask, info, &timer_sigq);

                if (unlikely(signr == SIGALRM))
                        posixtimer_rearm_itimer(tsk);
        }

        recalc_sigpending();
        if (!signr)
                return 0;

        if (unlikely(sig_kernel_stop(signr))) {
                /*
                 * Set a marker that we have dequeued a stop signal.  Our
                 * caller might release the siglock and then the pending
                 * stop signal it is about to process is no longer in the
                 * pending bitmasks, but must still be cleared by a SIGCONT
                 * (and overruled by a SIGKILL).  So those cases clear this
                 * shared flag after we've set it.  Note that this flag may
                 * remain set after the signal we return is ignored or
                 * handled.  That doesn't matter because its only purpose
                 * is to alert stop-signal processing code when another
                 * processor has come along and cleared the flag.
                 */
                current->jobctl |= JOBCTL_STOP_DEQUEUED;
        }

        if (IS_ENABLED(CONFIG_POSIX_TIMERS) && unlikely(timer_sigq)) {
                if (!posixtimer_deliver_signal(info, timer_sigq))
                        goto again;
        }

        return signr;
}
EXPORT_SYMBOL_GPL(dequeue_signal);

static int dequeue_synchronous_signal(kernel_siginfo_t *info)
{
        struct task_struct *tsk = current;
        struct sigpending *pending = &tsk->pending;
        struct sigqueue *q, *sync = NULL;

        /*
         * Might a synchronous signal be in the queue?
         */
        if (!((pending->signal.sig[0] & ~tsk->blocked.sig[0]) & SYNCHRONOUS_MASK))
                return 0;

        /*
         * Return the first synchronous signal in the queue.
         */
        list_for_each_entry(q, &pending->list, list) {
                /* Synchronous signals have a positive si_code */
                if ((q->info.si_code > SI_USER) &&
                    (sigmask(q->info.si_signo) & SYNCHRONOUS_MASK)) {
                        sync = q;
                        goto next;
                }
        }
        return 0;
next:
        /*
         * Check if there is another siginfo for the same signal.
         */
        list_for_each_entry_continue(q, &pending->list, list) {
                if (q->info.si_signo == sync->info.si_signo)
                        goto still_pending;
        }

        sigdelset(&pending->signal, sync->info.si_signo);
        recalc_sigpending();
still_pending:
        list_del_init(&sync->list);
        copy_siginfo(info, &sync->info);
        __sigqueue_free(sync);
        return info->si_signo;
}

/*
 * Tell a process that it has a new active signal..
 *
 * NOTE! we rely on the previous spin_lock to
 * lock interrupts for us! We can only be called with
 * "siglock" held, and the local interrupt must
 * have been disabled when that got acquired!
 *
 * No need to set need_resched since signal event passing
 * goes through ->blocked
 */
void signal_wake_up_state(struct task_struct *t, unsigned int state)
{
        lockdep_assert_held(&t->sighand->siglock);

        set_tsk_thread_flag(t, TIF_SIGPENDING);

        /*
         * TASK_WAKEKILL also means wake it up in the stopped/traced/killable
         * case. We don't check t->state here because there is a race with it
         * executing another processor and just now entering stopped state.
         * By using wake_up_state, we ensure the process will wake up and
         * handle its death signal.
         */
        if (!wake_up_state(t, state | TASK_INTERRUPTIBLE))
                kick_process(t);
}

static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q);

static void sigqueue_free_ignored(struct task_struct *tsk, struct sigqueue *q)
{
        if (likely(!(q->flags & SIGQUEUE_PREALLOC) || q->info.si_code != SI_TIMER))
                __sigqueue_free(q);
        else
                posixtimer_sig_ignore(tsk, q);
}

/* Remove signals in mask from the pending set and queue. */
static void flush_sigqueue_mask(struct task_struct *p, sigset_t *mask, struct sigpending *s)
{
        struct sigqueue *q, *n;
        sigset_t m;

        lockdep_assert_held(&p->sighand->siglock);

        sigandsets(&m, mask, &s->signal);
        if (sigisemptyset(&m))
                return;

        sigandnsets(&s->signal, &s->signal, mask);
        list_for_each_entry_safe(q, n, &s->list, list) {
                if (sigismember(mask, q->info.si_signo)) {
                        list_del_init(&q->list);
                        sigqueue_free_ignored(p, q);
                }
        }
}

static inline int is_si_special(const struct kernel_siginfo *info)
{
        return info <= SEND_SIG_PRIV;
}

static inline bool si_fromuser(const struct kernel_siginfo *info)
{
        return info == SEND_SIG_NOINFO ||
                (!is_si_special(info) && SI_FROMUSER(info));
}

/*
 * called with RCU read lock from check_kill_permission()
 */
static bool kill_ok_by_cred(struct task_struct *t)
{
        const struct cred *cred = current_cred();
        const struct cred *tcred = __task_cred(t);

        return uid_eq(cred->euid, tcred->suid) ||
               uid_eq(cred->euid, tcred->uid) ||
               uid_eq(cred->uid, tcred->suid) ||
               uid_eq(cred->uid, tcred->uid) ||
               ns_capable(tcred->user_ns, CAP_KILL);
}

/*
 * Bad permissions for sending the signal
 * - the caller must hold the RCU read lock
 */
static int check_kill_permission(int sig, struct kernel_siginfo *info,
                                 struct task_struct *t)
{
        struct pid *sid;
        int error;

        if (!valid_signal(sig))
                return -EINVAL;

        if (!si_fromuser(info))
                return 0;

        error = audit_signal_info(sig, t); /* Let audit system see the signal */
        if (error)
                return error;

        if (!same_thread_group(current, t) &&
            !kill_ok_by_cred(t)) {
                switch (sig) {
                case SIGCONT:
                        sid = task_session(t);
                        /*
                         * We don't return the error if sid == NULL. The
                         * task was unhashed, the caller must notice this.
                         */
                        if (!sid || sid == task_session(current))
                                break;
                        fallthrough;
                default:
                        return -EPERM;
                }
        }

        return security_task_kill(t, info, sig, NULL);
}

/**
 * ptrace_trap_notify - schedule trap to notify ptracer
 * @t: tracee wanting to notify tracer
 *
 * This function schedules sticky ptrace trap which is cleared on the next
 * TRAP_STOP to notify ptracer of an event.  @t must have been seized by
 * ptracer.
 *
 * If @t is running, STOP trap will be taken.  If trapped for STOP and
 * ptracer is listening for events, tracee is woken up so that it can
 * re-trap for the new event.  If trapped otherwise, STOP trap will be
 * eventually taken without returning to userland after the existing traps
 * are finished by PTRACE_CONT.
 *
 * CONTEXT:
 * Must be called with @task->sighand->siglock held.
 */
static void ptrace_trap_notify(struct task_struct *t)
{
        WARN_ON_ONCE(!(t->ptrace & PT_SEIZED));
        lockdep_assert_held(&t->sighand->siglock);

        task_set_jobctl_pending(t, JOBCTL_TRAP_NOTIFY);
        ptrace_signal_wake_up(t, t->jobctl & JOBCTL_LISTENING);
}

/*
 * Handle magic process-wide effects of stop/continue signals. Unlike
 * the signal actions, these happen immediately at signal-generation
 * time regardless of blocking, ignoring, or handling.  This does the
 * actual continuing for SIGCONT, but not the actual stopping for stop
 * signals. The process stop is done as a signal action for SIG_DFL.
 *
 * Returns true if the signal should be actually delivered, otherwise
 * it should be dropped.
 */
static bool prepare_signal(int sig, struct task_struct *p, bool force)
{
        struct signal_struct *signal = p->signal;
        struct task_struct *t;
        sigset_t flush;

        if (signal->flags & SIGNAL_GROUP_EXIT) {
                if (signal->core_state)
                        return sig == SIGKILL;
                /*
                 * The process is in the middle of dying, drop the signal.
                 */
                return false;
        } else if (sig_kernel_stop(sig)) {
                /*
                 * This is a stop signal.  Remove SIGCONT from all queues.
                 */
                siginitset(&flush, sigmask(SIGCONT));
                flush_sigqueue_mask(p, &flush, &signal->shared_pending);
                for_each_thread(p, t)
                        flush_sigqueue_mask(p, &flush, &t->pending);
        } else if (sig == SIGCONT) {
                unsigned int why;
                /*
                 * Remove all stop signals from all queues, wake all threads.
                 */
                siginitset(&flush, SIG_KERNEL_STOP_MASK);
                flush_sigqueue_mask(p, &flush, &signal->shared_pending);
                for_each_thread(p, t) {
                        flush_sigqueue_mask(p, &flush, &t->pending);
                        task_clear_jobctl_pending(t, JOBCTL_STOP_PENDING);
                        if (likely(!(t->ptrace & PT_SEIZED))) {
                                t->jobctl &= ~JOBCTL_STOPPED;
                                wake_up_state(t, __TASK_STOPPED);
                        } else
                                ptrace_trap_notify(t);
                }

                /*
                 * Notify the parent with CLD_CONTINUED if we were stopped.
                 *
                 * If we were in the middle of a group stop, we pretend it
                 * was already finished, and then continued. Since SIGCHLD
                 * doesn't queue we report only CLD_STOPPED, as if the next
                 * CLD_CONTINUED was dropped.
                 */
                why = 0;
                if (signal->flags & SIGNAL_STOP_STOPPED)
                        why |= SIGNAL_CLD_CONTINUED;
                else if (signal->group_stop_count)
                        why |= SIGNAL_CLD_STOPPED;

                if (why) {
                        /*
                         * The first thread which returns from do_signal_stop()
                         * will take ->siglock, notice SIGNAL_CLD_MASK, and
                         * notify its parent. See get_signal().
                         */
                        signal_set_stop_flags(signal, why | SIGNAL_STOP_CONTINUED);
                        signal->group_stop_count = 0;
                        signal->group_exit_code = 0;
                }
        }

        return !sig_ignored(p, sig, force);
}

/*
 * Test if P wants to take SIG.  After we've checked all threads with this,
 * it's equivalent to finding no threads not blocking SIG.  Any threads not
 * blocking SIG were ruled out because they are not running and already
 * have pending signals.  Such threads will dequeue from the shared queue
 * as soon as they're available, so putting the signal on the shared queue
 * will be equivalent to sending it to one such thread.
 */
static inline bool wants_signal(int sig, struct task_struct *p)
{
        if (sigismember(&p->blocked, sig))
                return false;

        if (p->flags & PF_EXITING)
                return false;

        if (sig == SIGKILL)
                return true;

        if (task_is_stopped_or_traced(p))
                return false;

        return task_curr(p) || !task_sigpending(p);
}

static void complete_signal(int sig, struct task_struct *p, enum pid_type type)
{
        struct signal_struct *signal = p->signal;
        struct task_struct *t;

        /*
         * Now find a thread we can wake up to take the signal off the queue.
         *
         * Try the suggested task first (may or may not be the main thread).
         */
        if (wants_signal(sig, p))
                t = p;
        else if ((type == PIDTYPE_PID) || thread_group_empty(p))
                /*
                 * There is just one thread and it does not need to be woken.
                 * It will dequeue unblocked signals before it runs again.
                 */
                return;
        else {
                /*
                 * Otherwise try to find a suitable thread.
                 */
                t = signal->curr_target;
                while (!wants_signal(sig, t)) {
                        t = next_thread(t);
                        if (t == signal->curr_target)
                                /*
                                 * No thread needs to be woken.
                                 * Any eligible threads will see
                                 * the signal in the queue soon.
                                 */
                                return;
                }
                signal->curr_target = t;
        }

        /*
         * Found a killable thread.  If the signal will be fatal,
         * then start taking the whole group down immediately.
         */
        if (sig_fatal(p, sig) &&
            (signal->core_state || !(signal->flags & SIGNAL_GROUP_EXIT)) &&
            !sigismember(&t->real_blocked, sig) &&
            (sig == SIGKILL || !p->ptrace)) {
                /*
                 * This signal will be fatal to the whole group.
                 */
                if (!sig_kernel_coredump(sig)) {
                        /*
                         * Start a group exit and wake everybody up.
                         * This way we don't have other threads
                         * running and doing things after a slower
                         * thread has the fatal signal pending.
                         */
                        signal->flags = SIGNAL_GROUP_EXIT;
                        signal->group_exit_code = sig;
                        signal->group_stop_count = 0;
                        __for_each_thread(signal, t) {
                                task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
                                sigaddset(&t->pending.signal, SIGKILL);
                                signal_wake_up(t, 1);
                        }
                        return;
                }
        }

        /*
         * The signal is already in the shared-pending queue.
         * Tell the chosen thread to wake up and dequeue it.
         */
        signal_wake_up(t, sig == SIGKILL);
        return;
}

static inline bool legacy_queue(struct sigpending *signals, int sig)
{
        return (sig < SIGRTMIN) && sigismember(&signals->signal, sig);
}

static int __send_signal_locked(int sig, struct kernel_siginfo *info,
                                struct task_struct *t, enum pid_type type, bool force)
{
        struct sigpending *pending;
        struct sigqueue *q;
        int override_rlimit;
        int ret = 0, result;

        lockdep_assert_held(&t->sighand->siglock);

        result = TRACE_SIGNAL_IGNORED;
        if (!prepare_signal(sig, t, force))
                goto ret;

        pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
        /*
         * Short-circuit ignored signals and support queuing
         * exactly one non-rt signal, so that we can get more
         * detailed information about the cause of the signal.
         */
        result = TRACE_SIGNAL_ALREADY_PENDING;
        if (legacy_queue(pending, sig))
                goto ret;

        result = TRACE_SIGNAL_DELIVERED;
        /*
         * Skip useless siginfo allocation for SIGKILL and kernel threads.
         */
        if ((sig == SIGKILL) || (t->flags & PF_KTHREAD))
                goto out_set;

        /*
         * Real-time signals must be queued if sent by sigqueue, or
         * some other real-time mechanism.  It is implementation
         * defined whether kill() does so.  We attempt to do so, on
         * the principle of least surprise, but since kill is not
         * allowed to fail with EAGAIN when low on memory we just
         * make sure at least one signal gets delivered and don't
         * pass on the info struct.
         */
        if (sig < SIGRTMIN)
                override_rlimit = (is_si_special(info) || info->si_code >= 0);
        else
                override_rlimit = 0;

        q = sigqueue_alloc(sig, t, GFP_ATOMIC, override_rlimit);

        if (q) {
                list_add_tail(&q->list, &pending->list);
                switch ((unsigned long) info) {
                case (unsigned long) SEND_SIG_NOINFO:
                        clear_siginfo(&q->info);
                        q->info.si_signo = sig;
                        q->info.si_errno = 0;
                        q->info.si_code = SI_USER;
                        q->info.si_pid = task_tgid_nr_ns(current,
                                                        task_active_pid_ns(t));
                        rcu_read_lock();
                        q->info.si_uid =
                                from_kuid_munged(task_cred_xxx(t, user_ns),
                                                 current_uid());
                        rcu_read_unlock();
                        break;
                case (unsigned long) SEND_SIG_PRIV:
                        clear_siginfo(&q->info);
                        q->info.si_signo = sig;
                        q->info.si_errno = 0;
                        q->info.si_code = SI_KERNEL;
                        q->info.si_pid = 0;
                        q->info.si_uid = 0;
                        break;
                default:
                        copy_siginfo(&q->info, info);
                        break;
                }
        } else if (!is_si_special(info) &&
                   sig >= SIGRTMIN && info->si_code != SI_USER) {
                /*
                 * Queue overflow, abort.  We may abort if the
                 * signal was rt and sent by user using something
                 * other than kill().
                 */
                result = TRACE_SIGNAL_OVERFLOW_FAIL;
                ret = -EAGAIN;
                goto ret;
        } else {
                /*
                 * This is a silent loss of information.  We still
                 * send the signal, but the *info bits are lost.
                 */
                result = TRACE_SIGNAL_LOSE_INFO;
        }

out_set:
        signalfd_notify(t, sig);
        sigaddset(&pending->signal, sig);

        /* Let multiprocess signals appear after on-going forks */
        if (type > PIDTYPE_TGID) {
                struct multiprocess_signals *delayed;
                hlist_for_each_entry(delayed, &t->signal->multiprocess, node) {
                        sigset_t *signal = &delayed->signal;
                        /* Can't queue both a stop and a continue signal */
                        if (sig == SIGCONT)
                                sigdelsetmask(signal, SIG_KERNEL_STOP_MASK);
                        else if (sig_kernel_stop(sig))
                                sigdelset(signal, SIGCONT);
                        sigaddset(signal, sig);
                }
        }

        complete_signal(sig, t, type);
ret:
        trace_signal_generate(sig, info, t, type != PIDTYPE_PID, result);
        return ret;
}

static inline bool has_si_pid_and_uid(struct kernel_siginfo *info)
{
        bool ret = false;
        switch (siginfo_layout(info->si_signo, info->si_code)) {
        case SIL_KILL:
        case SIL_CHLD:
        case SIL_RT:
                ret = true;
                break;
        case SIL_TIMER:
        case SIL_POLL:
        case SIL_FAULT:
        case SIL_FAULT_TRAPNO:
        case SIL_FAULT_MCEERR:
        case SIL_FAULT_BNDERR:
        case SIL_FAULT_PKUERR:
        case SIL_FAULT_PERF_EVENT:
        case SIL_SYS:
                ret = false;
                break;
        }
        return ret;
}

int send_signal_locked(int sig, struct kernel_siginfo *info,
                       struct task_struct *t, enum pid_type type)
{
        /* Should SIGKILL or SIGSTOP be received by a pid namespace init? */
        bool force = false;

        if (info == SEND_SIG_NOINFO) {
                /* Force if sent from an ancestor pid namespace */
                force = !task_pid_nr_ns(current, task_active_pid_ns(t));
        } else if (info == SEND_SIG_PRIV) {
                /* Don't ignore kernel generated signals */
                force = true;
        } else if (has_si_pid_and_uid(info)) {
                /* SIGKILL and SIGSTOP is special or has ids */
                struct user_namespace *t_user_ns;

                rcu_read_lock();
                t_user_ns = task_cred_xxx(t, user_ns);
                if (current_user_ns() != t_user_ns) {
                        kuid_t uid = make_kuid(current_user_ns(), info->si_uid);
                        info->si_uid = from_kuid_munged(t_user_ns, uid);
                }
                rcu_read_unlock();

                /* A kernel generated signal? */
                force = (info->si_code == SI_KERNEL);

                /* From an ancestor pid namespace? */
                if (!task_pid_nr_ns(current, task_active_pid_ns(t))) {
                        info->si_pid = 0;
                        force = true;
                }
        }
        return __send_signal_locked(sig, info, t, type, force);
}

static void print_fatal_signal(int signr)
{
        struct pt_regs *regs = task_pt_regs(current);
        struct file *exe_file;

        exe_file = get_task_exe_file(current);
        if (exe_file) {
                pr_info("%pD: %s: potentially unexpected fatal signal %d.\n",
                        exe_file, current->comm, signr);
                fput(exe_file);
        } else {
                pr_info("%s: potentially unexpected fatal signal %d.\n",
                        current->comm, signr);
        }

#if defined(__i386__) && !defined(__arch_um__)
        pr_info("code at %08lx: ", regs->ip);
        {
                int i;
                for (i = 0; i < 16; i++) {
                        unsigned char insn;

                        if (get_user(insn, (unsigned char *)(regs->ip + i)))
                                break;
                        pr_cont("%02x ", insn);
                }
        }
        pr_cont("\n");
#endif
        preempt_disable();
        show_regs(regs);
        preempt_enable();
}

static int __init setup_print_fatal_signals(char *str)
{
        get_option (&str, &print_fatal_signals);

        return 1;
}

__setup("print-fatal-signals=", setup_print_fatal_signals);

int do_send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p,
                        enum pid_type type)
{
        unsigned long flags;
        int ret = -ESRCH;

        if (lock_task_sighand(p, &flags)) {
                ret = send_signal_locked(sig, info, p, type);
                unlock_task_sighand(p, &flags);
        }

        return ret;
}

enum sig_handler {
        HANDLER_CURRENT, /* If reachable use the current handler */
        HANDLER_SIG_DFL, /* Always use SIG_DFL handler semantics */
        HANDLER_EXIT,         /* Only visible as the process exit code */
};

/*
 * Force a signal that the process can't ignore: if necessary
 * we unblock the signal and change any SIG_IGN to SIG_DFL.
 *
 * Note: If we unblock the signal, we always reset it to SIG_DFL,
 * since we do not want to have a signal handler that was blocked
 * be invoked when user space had explicitly blocked it.
 *
 * We don't want to have recursive SIGSEGV's etc, for example,
 * that is why we also clear SIGNAL_UNKILLABLE.
 */
static int
force_sig_info_to_task(struct kernel_siginfo *info, struct task_struct *t,
        enum sig_handler handler)
{
        unsigned long int flags;
        int ret, blocked, ignored;
        struct k_sigaction *action;
        int sig = info->si_signo;

        spin_lock_irqsave(&t->sighand->siglock, flags);
        action = &t->sighand->action[sig-1];
        ignored = action->sa.sa_handler == SIG_IGN;
        blocked = sigismember(&t->blocked, sig);
        if (blocked || ignored || (handler != HANDLER_CURRENT)) {
                action->sa.sa_handler = SIG_DFL;
                if (handler == HANDLER_EXIT)
                        action->sa.sa_flags |= SA_IMMUTABLE;
                if (blocked)
                        sigdelset(&t->blocked, sig);
        }
        /*
         * Don't clear SIGNAL_UNKILLABLE for traced tasks, users won't expect
         * debugging to leave init killable. But HANDLER_EXIT is always fatal.
         */
        if (action->sa.sa_handler == SIG_DFL &&
            (!t->ptrace || (handler == HANDLER_EXIT)))
                t->signal->flags &= ~SIGNAL_UNKILLABLE;
        ret = send_signal_locked(sig, info, t, PIDTYPE_PID);
        /* This can happen if the signal was already pending and blocked */
        if (!task_sigpending(t))
                signal_wake_up(t, 0);
        spin_unlock_irqrestore(&t->sighand->siglock, flags);

        return ret;
}

int force_sig_info(struct kernel_siginfo *info)
{
        return force_sig_info_to_task(info, current, HANDLER_CURRENT);
}

/*
 * Nuke all other threads in the group.
 */
int zap_other_threads(struct task_struct *p)
{
        struct task_struct *t;
        int count = 0;

        p->signal->group_stop_count = 0;

        for_other_threads(p, t) {
                task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK);
                count++;

                /* Don't bother with already dead threads */
                if (t->exit_state)
                        continue;
                sigaddset(&t->pending.signal, SIGKILL);
                signal_wake_up(t, 1);
        }

        return count;
}

struct sighand_struct *__lock_task_sighand(struct task_struct *tsk,
                                           unsigned long *flags)
{
        struct sighand_struct *sighand;

        rcu_read_lock();
        for (;;) {
                sighand = rcu_dereference(tsk->sighand);
                if (unlikely(sighand == NULL))
                        break;

                /*
                 * This sighand can be already freed and even reused, but
                 * we rely on SLAB_TYPESAFE_BY_RCU and sighand_ctor() which
                 * initializes ->siglock: this slab can't go away, it has
                 * the same object type, ->siglock can't be reinitialized.
                 *
                 * We need to ensure that tsk->sighand is still the same
                 * after we take the lock, we can race with de_thread() or
                 * __exit_signal(). In the latter case the next iteration
                 * must see ->sighand == NULL.
                 */
                spin_lock_irqsave(&sighand->siglock, *flags);
                if (likely(sighand == rcu_access_pointer(tsk->sighand)))
                        break;
                spin_unlock_irqrestore(&sighand->siglock, *flags);
        }
        rcu_read_unlock();

        return sighand;
}

#ifdef CONFIG_LOCKDEP
void lockdep_assert_task_sighand_held(struct task_struct *task)
{
        struct sighand_struct *sighand;

        rcu_read_lock();
        sighand = rcu_dereference(task->sighand);
        if (sighand)
                lockdep_assert_held(&sighand->siglock);
        else
                WARN_ON_ONCE(1);
        rcu_read_unlock();
}
#endif

/*
 * send signal info to all the members of a thread group or to the
 * individual thread if type == PIDTYPE_PID.
 */
int group_send_sig_info(int sig, struct kernel_siginfo *info,
                        struct task_struct *p, enum pid_type type)
{
        int ret;

        rcu_read_lock();
        ret = check_kill_permission(sig, info, p);
        rcu_read_unlock();

        if (!ret && sig)
                ret = do_send_sig_info(sig, info, p, type);

        return ret;
}

/*
 * __kill_pgrp_info() sends a signal to a process group: this is what the tty
 * control characters do (^C, ^Z etc)
 * - the caller must hold at least a readlock on tasklist_lock
 */
int __kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
        struct task_struct *p = NULL;
        int ret = -ESRCH;

        do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
                int err = group_send_sig_info(sig, info, p, PIDTYPE_PGID);
                /*
                 * If group_send_sig_info() succeeds at least once ret
                 * becomes 0 and after that the code below has no effect.
                 * Otherwise we return the last err or -ESRCH if this
                 * process group is empty.
                 */
                if (ret)
                        ret = err;
        } while_each_pid_task(pgrp, PIDTYPE_PGID, p);

        return ret;
}

static int kill_pid_info_type(int sig, struct kernel_siginfo *info,
                                struct pid *pid, enum pid_type type)
{
        int error = -ESRCH;
        struct task_struct *p;

        for (;;) {
                rcu_read_lock();
                p = pid_task(pid, PIDTYPE_PID);
                if (p)
                        error = group_send_sig_info(sig, info, p, type);
                rcu_read_unlock();
                if (likely(!p || error != -ESRCH))
                        return error;
                /*
                 * The task was unhashed in between, try again.  If it
                 * is dead, pid_task() will return NULL, if we race with
                 * de_thread() it will find the new leader.
                 */
        }
}

int kill_pid_info(int sig, struct kernel_siginfo *info, struct pid *pid)
{
        return kill_pid_info_type(sig, info, pid, PIDTYPE_TGID);
}

static int kill_proc_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
        int error;
        rcu_read_lock();
        error = kill_pid_info(sig, info, find_vpid(pid));
        rcu_read_unlock();
        return error;
}

static inline bool kill_as_cred_perm(const struct cred *cred,
                                     struct task_struct *target)
{
        const struct cred *pcred = __task_cred(target);

        return uid_eq(cred->euid, pcred->suid) ||
               uid_eq(cred->euid, pcred->uid) ||
               uid_eq(cred->uid, pcred->suid) ||
               uid_eq(cred->uid, pcred->uid);
}

/*
 * The usb asyncio usage of siginfo is wrong.  The glibc support
 * for asyncio which uses SI_ASYNCIO assumes the layout is SIL_RT.
 * AKA after the generic fields:
 *        kernel_pid_t        si_pid;
 *        kernel_uid32_t        si_uid;
 *        sigval_t        si_value;
 *
 * Unfortunately when usb generates SI_ASYNCIO it assumes the layout
 * after the generic fields is:
 *        void __user         *si_addr;
 *
 * This is a practical problem when there is a 64bit big endian kernel
 * and a 32bit userspace.  As the 32bit address will encoded in the low
 * 32bits of the pointer.  Those low 32bits will be stored at higher
 * address than appear in a 32 bit pointer.  So userspace will not
 * see the address it was expecting for it's completions.
 *
 * There is nothing in the encoding that can allow
 * copy_siginfo_to_user32 to detect this confusion of formats, so
 * handle this by requiring the caller of kill_pid_usb_asyncio to
 * notice when this situration takes place and to store the 32bit
 * pointer in sival_int, instead of sival_addr of the sigval_t addr
 * parameter.
 */
int kill_pid_usb_asyncio(int sig, int errno, sigval_t addr,
                         struct pid *pid, const struct cred *cred)
{
        struct kernel_siginfo info;
        struct task_struct *p;
        unsigned long flags;
        int ret = -EINVAL;

        if (!valid_signal(sig))
                return ret;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = errno;
        info.si_code = SI_ASYNCIO;
        *((sigval_t *)&info.si_pid) = addr;

        rcu_read_lock();
        p = pid_task(pid, PIDTYPE_PID);
        if (!p) {
                ret = -ESRCH;
                goto out_unlock;
        }
        if (!kill_as_cred_perm(cred, p)) {
                ret = -EPERM;
                goto out_unlock;
        }
        ret = security_task_kill(p, &info, sig, cred);
        if (ret)
                goto out_unlock;

        if (sig) {
                if (lock_task_sighand(p, &flags)) {
                        ret = __send_signal_locked(sig, &info, p, PIDTYPE_TGID, false);
                        unlock_task_sighand(p, &flags);
                } else
                        ret = -ESRCH;
        }
out_unlock:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(kill_pid_usb_asyncio);

/*
 * kill_something_info() interprets pid in interesting ways just like kill(2).
 *
 * POSIX specifies that kill(-1,sig) is unspecified, but what we have
 * is probably wrong.  Should make it like BSD or SYSV.
 */

static int kill_something_info(int sig, struct kernel_siginfo *info, pid_t pid)
{
        int ret;

        if (pid > 0)
                return kill_proc_info(sig, info, pid);

        /* -INT_MIN is undefined.  Exclude this case to avoid a UBSAN warning */
        if (pid == INT_MIN)
                return -ESRCH;

        read_lock(&tasklist_lock);
        if (pid != -1) {
                ret = __kill_pgrp_info(sig, info,
                                pid ? find_vpid(-pid) : task_pgrp(current));
        } else {
                int retval = 0, count = 0;
                struct task_struct * p;

                for_each_process(p) {
                        if (task_pid_vnr(p) > 1 &&
                                        !same_thread_group(p, current)) {
                                int err = group_send_sig_info(sig, info, p,
                                                              PIDTYPE_MAX);
                                ++count;
                                if (err != -EPERM)
                                        retval = err;
                        }
                }
                ret = count ? retval : -ESRCH;
        }
        read_unlock(&tasklist_lock);

        return ret;
}

/*
 * These are for backward compatibility with the rest of the kernel source.
 */

int send_sig_info(int sig, struct kernel_siginfo *info, struct task_struct *p)
{
        /*
         * Make sure legacy kernel users don't send in bad values
         * (normal paths check this in check_kill_permission).
         */
        if (!valid_signal(sig))
                return -EINVAL;

        return do_send_sig_info(sig, info, p, PIDTYPE_PID);
}
EXPORT_SYMBOL(send_sig_info);

#define __si_special(priv) \
        ((priv) ? SEND_SIG_PRIV : SEND_SIG_NOINFO)

int
send_sig(int sig, struct task_struct *p, int priv)
{
        return send_sig_info(sig, __si_special(priv), p);
}
EXPORT_SYMBOL(send_sig);

void force_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info(&info);
}
EXPORT_SYMBOL(force_sig);

void force_fatal_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info_to_task(&info, current, HANDLER_SIG_DFL);
}

void force_exit_sig(int sig)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code = SI_KERNEL;
        info.si_pid = 0;
        info.si_uid = 0;
        force_sig_info_to_task(&info, current, HANDLER_EXIT);
}

/*
 * When things go south during signal handling, we
 * will force a SIGSEGV. And if the signal that caused
 * the problem was already a SIGSEGV, we'll want to
 * make sure we don't even try to deliver the signal..
 */
void force_sigsegv(int sig)
{
        if (sig == SIGSEGV)
                force_fatal_sig(SIGSEGV);
        else
                force_sig(SIGSEGV);
}

int force_sig_fault_to_task(int sig, int code, void __user *addr,
                            struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        return force_sig_info_to_task(&info, t, HANDLER_CURRENT);
}

int force_sig_fault(int sig, int code, void __user *addr)
{
        return force_sig_fault_to_task(sig, code, addr, current);
}

int send_sig_fault(int sig, int code, void __user *addr, struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        return send_sig_info(info.si_signo, &info, t);
}

int force_sig_mceerr(int code, void __user *addr, short lsb)
{
        struct kernel_siginfo info;

        WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
        clear_siginfo(&info);
        info.si_signo = SIGBUS;
        info.si_errno = 0;
        info.si_code = code;
        info.si_addr = addr;
        info.si_addr_lsb = lsb;
        return force_sig_info(&info);
}

int send_sig_mceerr(int code, void __user *addr, short lsb, struct task_struct *t)
{
        struct kernel_siginfo info;

        WARN_ON((code != BUS_MCEERR_AO) && (code != BUS_MCEERR_AR));
        clear_siginfo(&info);
        info.si_signo = SIGBUS;
        info.si_errno = 0;
        info.si_code = code;
        info.si_addr = addr;
        info.si_addr_lsb = lsb;
        return send_sig_info(info.si_signo, &info, t);
}
EXPORT_SYMBOL(send_sig_mceerr);

int force_sig_bnderr(void __user *addr, void __user *lower, void __user *upper)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSEGV;
        info.si_errno = 0;
        info.si_code  = SEGV_BNDERR;
        info.si_addr  = addr;
        info.si_lower = lower;
        info.si_upper = upper;
        return force_sig_info(&info);
}

#ifdef SEGV_PKUERR
int force_sig_pkuerr(void __user *addr, u32 pkey)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSEGV;
        info.si_errno = 0;
        info.si_code  = SEGV_PKUERR;
        info.si_addr  = addr;
        info.si_pkey  = pkey;
        return force_sig_info(&info);
}
#endif

int send_sig_perf(void __user *addr, u32 type, u64 sig_data)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo     = SIGTRAP;
        info.si_errno     = 0;
        info.si_code      = TRAP_PERF;
        info.si_addr      = addr;
        info.si_perf_data = sig_data;
        info.si_perf_type = type;

        /*
         * Signals generated by perf events should not terminate the whole
         * process if SIGTRAP is blocked, however, delivering the signal
         * asynchronously is better than not delivering at all. But tell user
         * space if the signal was asynchronous, so it can clearly be
         * distinguished from normal synchronous ones.
         */
        info.si_perf_flags = sigismember(&current->blocked, info.si_signo) ?
                                     TRAP_PERF_FLAG_ASYNC :
                                     0;

        return send_sig_info(info.si_signo, &info, current);
}

/**
 * force_sig_seccomp - signals the task to allow in-process syscall emulation
 * @syscall: syscall number to send to userland
 * @reason: filter-supplied reason code to send to userland (via si_errno)
 * @force_coredump: true to trigger a coredump
 *
 * Forces a SIGSYS with a code of SYS_SECCOMP and related sigsys info.
 */
int force_sig_seccomp(int syscall, int reason, bool force_coredump)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGSYS;
        info.si_code = SYS_SECCOMP;
        info.si_call_addr = (void __user *)KSTK_EIP(current);
        info.si_errno = reason;
        info.si_arch = syscall_get_arch(current);
        info.si_syscall = syscall;
        return force_sig_info_to_task(&info, current,
                force_coredump ? HANDLER_EXIT : HANDLER_CURRENT);
}

/* For the crazy architectures that include trap information in
 * the errno field, instead of an actual errno value.
 */
int force_sig_ptrace_errno_trap(int errno, void __user *addr)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = SIGTRAP;
        info.si_errno = errno;
        info.si_code  = TRAP_HWBKPT;
        info.si_addr  = addr;
        return force_sig_info(&info);
}

/* For the rare architectures that include trap information using
 * si_trapno.
 */
int force_sig_fault_trapno(int sig, int code, void __user *addr, int trapno)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        info.si_trapno = trapno;
        return force_sig_info(&info);
}

/* For the rare architectures that include trap information using
 * si_trapno.
 */
int send_sig_fault_trapno(int sig, int code, void __user *addr, int trapno,
                          struct task_struct *t)
{
        struct kernel_siginfo info;

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        info.si_code  = code;
        info.si_addr  = addr;
        info.si_trapno = trapno;
        return send_sig_info(info.si_signo, &info, t);
}

static int kill_pgrp_info(int sig, struct kernel_siginfo *info, struct pid *pgrp)
{
        int ret;
        read_lock(&tasklist_lock);
        ret = __kill_pgrp_info(sig, info, pgrp);
        read_unlock(&tasklist_lock);
        return ret;
}

int kill_pgrp(struct pid *pid, int sig, int priv)
{
        return kill_pgrp_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pgrp);

int kill_pid(struct pid *pid, int sig, int priv)
{
        return kill_pid_info(sig, __si_special(priv), pid);
}
EXPORT_SYMBOL(kill_pid);

#ifdef CONFIG_POSIX_TIMERS
/*
 * These functions handle POSIX timer signals. POSIX timers use
 * preallocated sigqueue structs for sending signals.
 */
static void __flush_itimer_signals(struct sigpending *pending)
{
        sigset_t signal, retain;
        struct sigqueue *q, *n;

        signal = pending->signal;
        sigemptyset(&retain);

        list_for_each_entry_safe(q, n, &pending->list, list) {
                int sig = q->info.si_signo;

                if (likely(q->info.si_code != SI_TIMER)) {
                        sigaddset(&retain, sig);
                } else {
                        sigdelset(&signal, sig);
                        list_del_init(&q->list);
                        __sigqueue_free(q);
                }
        }

        sigorsets(&pending->signal, &signal, &retain);
}

void flush_itimer_signals(void)
{
        struct task_struct *tsk = current;

        guard(spinlock_irqsave)(&tsk->sighand->siglock);
        __flush_itimer_signals(&tsk->pending);
        __flush_itimer_signals(&tsk->signal->shared_pending);
}

bool posixtimer_init_sigqueue(struct sigqueue *q)
{
        struct ucounts *ucounts = sig_get_ucounts(current, -1, 0);

        if (!ucounts)
                return false;
        clear_siginfo(&q->info);
        __sigqueue_init(q, ucounts, SIGQUEUE_PREALLOC);
        return true;
}

static void posixtimer_queue_sigqueue(struct sigqueue *q, struct task_struct *t, enum pid_type type)
{
        struct sigpending *pending;
        int sig = q->info.si_signo;

        signalfd_notify(t, sig);
        pending = (type != PIDTYPE_PID) ? &t->signal->shared_pending : &t->pending;
        list_add_tail(&q->list, &pending->list);
        sigaddset(&pending->signal, sig);
        complete_signal(sig, t, type);
}

/*
 * This function is used by POSIX timers to deliver a timer signal.
 * Where type is PIDTYPE_PID (such as for timers with SIGEV_THREAD_ID
 * set), the signal must be delivered to the specific thread (queues
 * into t->pending).
 *
 * Where type is not PIDTYPE_PID, signals must be delivered to the
 * process. In this case, prefer to deliver to current if it is in
 * the same thread group as the target process and its sighand is
 * stable, which avoids unnecessarily waking up a potentially idle task.
 */
static inline struct task_struct *posixtimer_get_target(struct k_itimer *tmr)
{
        struct task_struct *t = pid_task(tmr->it_pid, tmr->it_pid_type);

        if (t && tmr->it_pid_type != PIDTYPE_PID &&
            same_thread_group(t, current) && !current->exit_state)
                t = current;
        return t;
}

void posixtimer_send_sigqueue(struct k_itimer *tmr)
{
        struct sigqueue *q = &tmr->sigq;
        int sig = q->info.si_signo;
        struct task_struct *t;
        unsigned long flags;
        int result;

        guard(rcu)();

        t = posixtimer_get_target(tmr);
        if (!t)
                return;

        if (!likely(lock_task_sighand(t, &flags)))
                return;

        /*
         * Update @tmr::sigqueue_seq for posix timer signals with sighand
         * locked to prevent a race against dequeue_signal().
         */
        tmr->it_sigqueue_seq = tmr->it_signal_seq;

        /*
         * Set the signal delivery status under sighand lock, so that the
         * ignored signal handling can distinguish between a periodic and a
         * non-periodic timer.
         */
        tmr->it_sig_periodic = tmr->it_status == POSIX_TIMER_REQUEUE_PENDING;

        if (!prepare_signal(sig, t, false)) {
                result = TRACE_SIGNAL_IGNORED;

                if (!list_empty(&q->list)) {
                        /*
                         * The signal was ignored and blocked. The timer
                         * expiry queued it because blocked signals are
                         * queued independent of the ignored state.
                         *
                         * The unblocking set SIGPENDING, but the signal
                         * was not yet dequeued from the pending list.
                         * So prepare_signal() sees unblocked and ignored,
                         * which ends up here. Leave it queued like a
                         * regular signal.
                         *
                         * The same happens when the task group is exiting
                         * and the signal is already queued.
                         * prepare_signal() treats SIGNAL_GROUP_EXIT as
                         * ignored independent of its queued state. This
                         * gets cleaned up in __exit_signal().
                         */
                        goto out;
                }

                /* Periodic timers with SIG_IGN are queued on the ignored list */
                if (tmr->it_sig_periodic) {
                        /*
                         * Already queued means the timer was rearmed after
                         * the previous expiry got it on the ignore list.
                         * Nothing to do for that case.
                         */
                        if (hlist_unhashed(&tmr->ignored_list)) {
                                /*
                                 * Take a signal reference and queue it on
                                 * the ignored list.
                                 */
                                posixtimer_sigqueue_getref(q);
                                posixtimer_sig_ignore(t, q);
                        }
                } else if (!hlist_unhashed(&tmr->ignored_list)) {
                        /*
                         * Covers the case where a timer was periodic and
                         * then the signal was ignored. Later it was rearmed
                         * as oneshot timer. The previous signal is invalid
                         * now, and this oneshot signal has to be dropped.
                         * Remove it from the ignored list and drop the
                         * reference count as the signal is not longer
                         * queued.
                         */
                        hlist_del_init(&tmr->ignored_list);
                        posixtimer_putref(tmr);
                }
                goto out;
        }

        if (unlikely(!list_empty(&q->list))) {
                /* This holds a reference count already */
                result = TRACE_SIGNAL_ALREADY_PENDING;
                goto out;
        }

        /*
         * If the signal is on the ignore list, it got blocked after it was
         * ignored earlier. But nothing lifted the ignore. Move it back to
         * the pending list to be consistent with the regular signal
         * handling. This already holds a reference count.
         *
         * If it's not on the ignore list acquire a reference count.
         */
        if (likely(hlist_unhashed(&tmr->ignored_list)))
                posixtimer_sigqueue_getref(q);
        else
                hlist_del_init(&tmr->ignored_list);

        posixtimer_queue_sigqueue(q, t, tmr->it_pid_type);
        result = TRACE_SIGNAL_DELIVERED;
out:
        trace_signal_generate(sig, &q->info, t, tmr->it_pid_type != PIDTYPE_PID, result);
        unlock_task_sighand(t, &flags);
}

static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q)
{
        struct k_itimer *tmr = container_of(q, struct k_itimer, sigq);

        /*
         * If the timer is marked deleted already or the signal originates
         * from a non-periodic timer, then just drop the reference
         * count. Otherwise queue it on the ignored list.
         */
        if (posixtimer_valid(tmr) && tmr->it_sig_periodic)
                hlist_add_head(&tmr->ignored_list, &tsk->signal->ignored_posix_timers);
        else
                posixtimer_putref(tmr);
}

static void posixtimer_sig_unignore(struct task_struct *tsk, int sig)
{
        struct hlist_head *head = &tsk->signal->ignored_posix_timers;
        struct hlist_node *tmp;
        struct k_itimer *tmr;

        if (likely(hlist_empty(head)))
                return;

        /*
         * Rearming a timer with sighand lock held is not possible due to
         * lock ordering vs. tmr::it_lock. Just stick the sigqueue back and
         * let the signal delivery path deal with it whether it needs to be
         * rearmed or not. This cannot be decided here w/o dropping sighand
         * lock and creating a loop retry horror show.
         */
        hlist_for_each_entry_safe(tmr, tmp , head, ignored_list) {
                struct task_struct *target;

                /*
                 * tmr::sigq.info.si_signo is immutable, so accessing it
                 * without holding tmr::it_lock is safe.
                 */
                if (tmr->sigq.info.si_signo != sig)
                        continue;

                hlist_del_init(&tmr->ignored_list);

                /* This should never happen and leaks a reference count */
                if (WARN_ON_ONCE(!list_empty(&tmr->sigq.list)))
                        continue;

                /*
                 * Get the target for the signal. If target is a thread and
                 * has exited by now, drop the reference count.
                 */
                guard(rcu)();
                target = posixtimer_get_target(tmr);
                if (target)
                        posixtimer_queue_sigqueue(&tmr->sigq, target, tmr->it_pid_type);
                else
                        posixtimer_putref(tmr);
        }
}
#else /* CONFIG_POSIX_TIMERS */
static inline void posixtimer_sig_ignore(struct task_struct *tsk, struct sigqueue *q) { }
static inline void posixtimer_sig_unignore(struct task_struct *tsk, int sig) { }
#endif /* !CONFIG_POSIX_TIMERS */

void do_notify_pidfd(struct task_struct *task)
{
        struct pid *pid = task_pid(task);

        WARN_ON(task->exit_state == 0);

        __wake_up(&pid->wait_pidfd, TASK_NORMAL, 0,
                        poll_to_key(EPOLLIN | EPOLLRDNORM));
}

/*
 * Let a parent know about the death of a child.
 * For a stopped/continued status change, use do_notify_parent_cldstop instead.
 *
 * Returns true if our parent ignored us and so we've switched to
 * self-reaping.
 */
bool do_notify_parent(struct task_struct *tsk, int sig)
{
        struct kernel_siginfo info;
        unsigned long flags;
        struct sighand_struct *psig;
        bool autoreap = false;
        u64 utime, stime;

        WARN_ON_ONCE(sig == -1);

        /* do_notify_parent_cldstop should have been called instead.  */
        WARN_ON_ONCE(task_is_stopped_or_traced(tsk));

        WARN_ON_ONCE(!tsk->ptrace &&
               (tsk->group_leader != tsk || !thread_group_empty(tsk)));

        /* ptraced, or group-leader without sub-threads */
        do_notify_pidfd(tsk);

        if (sig != SIGCHLD) {
                /*
                 * This is only possible if parent == real_parent.
                 * Check if it has changed security domain.
                 */
                if (tsk->parent_exec_id != READ_ONCE(tsk->parent->self_exec_id))
                        sig = SIGCHLD;
        }

        clear_siginfo(&info);
        info.si_signo = sig;
        info.si_errno = 0;
        /*
         * We are under tasklist_lock here so our parent is tied to
         * us and cannot change.
         *
         * task_active_pid_ns will always return the same pid namespace
         * until a task passes through release_task.
         *
         * write_lock() currently calls preempt_disable() which is the
         * same as rcu_read_lock(), but according to Oleg, this is not
         * correct to rely on this
         */
        rcu_read_lock();
        info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(tsk->parent));
        info.si_uid = from_kuid_munged(task_cred_xxx(tsk->parent, user_ns),
                                       task_uid(tsk));
        rcu_read_unlock();

        task_cputime(tsk, &utime, &stime);
        info.si_utime = nsec_to_clock_t(utime + tsk->signal->utime);
        info.si_stime = nsec_to_clock_t(stime + tsk->signal->stime);

        info.si_status = tsk->exit_code & 0x7f;
        if (tsk->exit_code & 0x80)
                info.si_code = CLD_DUMPED;
        else if (tsk->exit_code & 0x7f)
                info.si_code = CLD_KILLED;
        else {
                info.si_code = CLD_EXITED;
                info.si_status = tsk->exit_code >> 8;
        }

        psig = tsk->parent->sighand;
        spin_lock_irqsave(&psig->siglock, flags);
        if (!tsk->ptrace && sig == SIGCHLD &&
            (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN ||
             (psig->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDWAIT))) {
                /*
                 * We are exiting and our parent doesn't care.  POSIX.1
                 * defines special semantics for setting SIGCHLD to SIG_IGN
                 * or setting the SA_NOCLDWAIT flag: we should be reaped
                 * automatically and not left for our parent's wait4 call.
                 * Rather than having the parent do it as a magic kind of
                 * signal handler, we just set this to tell do_exit that we
                 * can be cleaned up without becoming a zombie.  Note that
                 * we still call __wake_up_parent in this case, because a
                 * blocked sys_wait4 might now return -ECHILD.
                 *
                 * Whether we send SIGCHLD or not for SA_NOCLDWAIT
                 * is implementation-defined: we do (if you don't want
                 * it, just use SIG_IGN instead).
                 */
                autoreap = true;
                if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN)
                        sig = 0;
        }
        /*
         * Send with __send_signal as si_pid and si_uid are in the
         * parent's namespaces.
         */
        if (valid_signal(sig) && sig)
                __send_signal_locked(sig, &info, tsk->parent, PIDTYPE_TGID, false);
        __wake_up_parent(tsk, tsk->parent);
        spin_unlock_irqrestore(&psig->siglock, flags);

        return autoreap;
}

/**
 * do_notify_parent_cldstop - notify parent of stopped/continued state change
 * @tsk: task reporting the state change
 * @for_ptracer: the notification is for ptracer
 * @why: CLD_{CONTINUED|STOPPED|TRAPPED} to report
 *
 * Notify @tsk's parent that the stopped/continued state has changed.  If
 * @for_ptracer is %false, @tsk's group leader notifies to its real parent.
 * If %true, @tsk reports to @tsk->parent which should be the ptracer.
 *
 * CONTEXT:
 * Must be called with tasklist_lock at least read locked.
 */
static void do_notify_parent_cldstop(struct task_struct *tsk,
                                     bool for_ptracer, int why)
{
        struct kernel_siginfo info;
        unsigned long flags;
        struct task_struct *parent;
        struct sighand_struct *sighand;
        u64 utime, stime;

        if (for_ptracer) {
                parent = tsk->parent;
        } else {
                tsk = tsk->group_leader;
                parent = tsk->real_parent;
        }

        clear_siginfo(&info);
        info.si_signo = SIGCHLD;
        info.si_errno = 0;
        /*
         * see comment in do_notify_parent() about the following 4 lines
         */
        rcu_read_lock();
        info.si_pid = task_pid_nr_ns(tsk, task_active_pid_ns(parent));
        info.si_uid = from_kuid_munged(task_cred_xxx(parent, user_ns), task_uid(tsk));
        rcu_read_unlock();

        task_cputime(tsk, &utime, &stime);
        info.si_utime = nsec_to_clock_t(utime);
        info.si_stime = nsec_to_clock_t(stime);

         info.si_code = why;
         switch (why) {
         case CLD_CONTINUED:
                 info.si_status = SIGCONT;
                 break;
         case CLD_STOPPED:
                 info.si_status = tsk->signal->group_exit_code & 0x7f;
                 break;
         case CLD_TRAPPED:
                 info.si_status = tsk->exit_code & 0x7f;
                 break;
         default:
                 BUG();
         }

        sighand = parent->sighand;
        spin_lock_irqsave(&sighand->siglock, flags);
        if (sighand->action[SIGCHLD-1].sa.sa_handler != SIG_IGN &&
            !(sighand->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDSTOP))
                send_signal_locked(SIGCHLD, &info, parent, PIDTYPE_TGID);
        /*
         * Even if SIGCHLD is not generated, we must wake up wait4 calls.
         */
        __wake_up_parent(tsk, parent);
        spin_unlock_irqrestore(&sighand->siglock, flags);
}

/*
 * This must be called with current->sighand->siglock held.
 *
 * This should be the path for all ptrace stops.
 * We always set current->last_siginfo while stopped here.
 * That makes it a way to test a stopped process for
 * being ptrace-stopped vs being job-control-stopped.
 *
 * Returns the signal the ptracer requested the code resume
 * with.  If the code did not stop because the tracer is gone,
 * the stop signal remains unchanged unless clear_code.
 */
static int ptrace_stop(int exit_code, int why, unsigned long message,
                       kernel_siginfo_t *info)
        __releases(&current->sighand->siglock)
        __acquires(&current->sighand->siglock)
{
        bool gstop_done = false;

        if (arch_ptrace_stop_needed()) {
                /*
                 * The arch code has something special to do before a
                 * ptrace stop.  This is allowed to block, e.g. for faults
                 * on user stack pages.  We can't keep the siglock while
                 * calling arch_ptrace_stop, so we must release it now.
                 * To preserve proper semantics, we must do this before
                 * any signal bookkeeping like checking group_stop_count.
                 */
                spin_unlock_irq(&current->sighand->siglock);
                arch_ptrace_stop();
                spin_lock_irq(&current->sighand->siglock);
        }

        /*
         * After this point ptrace_signal_wake_up or signal_wake_up
         * will clear TASK_TRACED if ptrace_unlink happens or a fatal
         * signal comes in.  Handle previous ptrace_unlinks and fatal
         * signals here to prevent ptrace_stop sleeping in schedule.
         */
        if (!current->ptrace || __fatal_signal_pending(current))
                return exit_code;

        set_special_state(TASK_TRACED);
        current->jobctl |= JOBCTL_TRACED;

        /*
         * We're committing to trapping.  TRACED should be visible before
         * TRAPPING is cleared; otherwise, the tracer might fail do_wait().
         * Also, transition to TRACED and updates to ->jobctl should be
         * atomic with respect to siglock and should be done after the arch
         * hook as siglock is released and regrabbed across it.
         *
         *     TRACER                                    TRACEE
         *
         *     ptrace_attach()
         * [L]   wait_on_bit(JOBCTL_TRAPPING)        [S] set_special_state(TRACED)
         *     do_wait()
         *       set_current_state()                smp_wmb();
         *       ptrace_do_wait()
         *         wait_task_stopped()
         *           task_stopped_code()
         * [L]         task_is_traced()                [S] task_clear_jobctl_trapping();
         */
        smp_wmb();

        current->ptrace_message = message;
        current->last_siginfo = info;
        current->exit_code = exit_code;

        /*
         * If @why is CLD_STOPPED, we're trapping to participate in a group
         * stop.  Do the bookkeeping.  Note that if SIGCONT was delievered
         * across siglock relocks since INTERRUPT was scheduled, PENDING
         * could be clear now.  We act as if SIGCONT is received after
         * TASK_TRACED is entered - ignore it.
         */
        if (why == CLD_STOPPED && (current->jobctl & JOBCTL_STOP_PENDING))
                gstop_done = task_participate_group_stop(current);

        /* any trap clears pending STOP trap, STOP trap clears NOTIFY */
        task_clear_jobctl_pending(current, JOBCTL_TRAP_STOP);
        if (info && info->si_code >> 8 == PTRACE_EVENT_STOP)
                task_clear_jobctl_pending(current, JOBCTL_TRAP_NOTIFY);

        /* entering a trap, clear TRAPPING */
        task_clear_jobctl_trapping(current);

        spin_unlock_irq(&current->sighand->siglock);
        read_lock(&tasklist_lock);
        /*
         * Notify parents of the stop.
         *
         * While ptraced, there are two parents - the ptracer and
         * the real_parent of the group_leader.  The ptracer should
         * know about every stop while the real parent is only
         * interested in the completion of group stop.  The states
         * for the two don't interact with each other.  Notify
         * separately unless they're gonna be duplicates.
         */
        if (current->ptrace)
                do_notify_parent_cldstop(current, true, why);
        if (gstop_done && (!current->ptrace || ptrace_reparented(current)))
                do_notify_parent_cldstop(current, false, why);

        /*
         * The previous do_notify_parent_cldstop() invocation woke ptracer.
         * One a PREEMPTION kernel this can result in preemption requirement
         * which will be fulfilled after read_unlock() and the ptracer will be
         * put on the CPU.
         * The ptracer is in wait_task_inactive(, __TASK_TRACED) waiting for
         * this task wait in schedule(). If this task gets preempted then it
         * remains enqueued on the runqueue. The ptracer will observe this and
         * then sleep for a delay of one HZ tick. In the meantime this task
         * gets scheduled, enters schedule() and will wait for the ptracer.
         *
         * This preemption point is not bad from a correctness point of
         * view but extends the runtime by one HZ tick time due to the
         * ptracer's sleep.  The preempt-disable section ensures that there
         * will be no preemption between unlock and schedule() and so
         * improving the performance since the ptracer will observe that
         * the tracee is scheduled out once it gets on the CPU.
         *
         * On PREEMPT_RT locking tasklist_lock does not disable preemption.
         * Therefore the task can be preempted after do_notify_parent_cldstop()
         * before unlocking tasklist_lock so there is no benefit in doing this.
         *
         * In fact disabling preemption is harmful on PREEMPT_RT because
         * the spinlock_t in cgroup_enter_frozen() must not be acquired
         * with preemption disabled due to the 'sleeping' spinlock
         * substitution of RT.
         */
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_disable();
        read_unlock(&tasklist_lock);
        cgroup_enter_frozen();
        if (!IS_ENABLED(CONFIG_PREEMPT_RT))
                preempt_enable_no_resched();
        schedule();
        cgroup_leave_frozen(true);

        /*
         * We are back.  Now reacquire the siglock before touching
         * last_siginfo, so that we are sure to have synchronized with
         * any signal-sending on another CPU that wants to examine it.
         */
        spin_lock_irq(&current->sighand->siglock);
        exit_code = current->exit_code;
        current->last_siginfo = NULL;
        current->ptrace_message = 0;
        current->exit_code = 0;

        /* LISTENING can be set only during STOP traps, clear it */
        current->jobctl &= ~(JOBCTL_LISTENING | JOBCTL_PTRACE_FROZEN);

        /*
         * Queued signals ignored us while we were stopped for tracing.
         * So check for any that we should take before resuming user mode.
         * This sets TIF_SIGPENDING, but never clears it.
         */
        recalc_sigpending_tsk(current);
        return exit_code;
}

static int ptrace_do_notify(int signr, int exit_code, int why, unsigned long message)
{
        kernel_siginfo_t info;

        clear_siginfo(&info);
        info.si_signo = signr;
        info.si_code = exit_code;
        info.si_pid = task_pid_vnr(current);
        info.si_uid = from_kuid_munged(current_user_ns(), current_uid());

        /* Let the debugger run.  */
        return ptrace_stop(exit_code, why, message, &info);
}

int ptrace_notify(int exit_code, unsigned long message)
{
        int signr;

        BUG_ON((exit_code & (0x7f | ~0xffff)) != SIGTRAP);
        if (unlikely(task_work_pending(current)))
                task_work_run();

        spin_lock_irq(&current->sighand->siglock);
        signr = ptrace_do_notify(SIGTRAP, exit_code, CLD_TRAPPED, message);
        spin_unlock_irq(&current->sighand->siglock);
        return signr;
}

/**
 * do_signal_stop - handle group stop for SIGSTOP and other stop signals
 * @signr: signr causing group stop if initiating
 *
 * If %JOBCTL_STOP_PENDING is not set yet, initiate group stop with @signr
 * and participate in it.  If already set, participate in the existing
 * group stop.  If participated in a group stop (and thus slept), %true is
 * returned with siglock released.
 *
 * If ptraced, this function doesn't handle stop itself.  Instead,
 * %JOBCTL_TRAP_STOP is scheduled and %false is returned with siglock
 * untouched.  The caller must ensure that INTERRUPT trap handling takes
 * places afterwards.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held, which is released
 * on %true return.
 *
 * RETURNS:
 * %false if group stop is already cancelled or ptrace trap is scheduled.
 * %true if participated in group stop.
 */
static bool do_signal_stop(int signr)
        __releases(&current->sighand->siglock)
{
        struct signal_struct *sig = current->signal;

        if (!(current->jobctl & JOBCTL_STOP_PENDING)) {
                unsigned long gstop = JOBCTL_STOP_PENDING | JOBCTL_STOP_CONSUME;
                struct task_struct *t;

                /* signr will be recorded in task->jobctl for retries */
                WARN_ON_ONCE(signr & ~JOBCTL_STOP_SIGMASK);

                if (!likely(current->jobctl & JOBCTL_STOP_DEQUEUED) ||
                    unlikely(sig->flags & SIGNAL_GROUP_EXIT) ||
                    unlikely(sig->group_exec_task))
                        return false;
                /*
                 * There is no group stop already in progress.  We must
                 * initiate one now.
                 *
                 * While ptraced, a task may be resumed while group stop is
                 * still in effect and then receive a stop signal and
                 * initiate another group stop.  This deviates from the
                 * usual behavior as two consecutive stop signals can't
                 * cause two group stops when !ptraced.  That is why we
                 * also check !task_is_stopped(t) below.
                 *
                 * The condition can be distinguished by testing whether
                 * SIGNAL_STOP_STOPPED is already set.  Don't generate
                 * group_exit_code in such case.
                 *
                 * This is not necessary for SIGNAL_STOP_CONTINUED because
                 * an intervening stop signal is required to cause two
                 * continued events regardless of ptrace.
                 */
                if (!(sig->flags & SIGNAL_STOP_STOPPED))
                        sig->group_exit_code = signr;

                sig->group_stop_count = 0;
                if (task_set_jobctl_pending(current, signr | gstop))
                        sig->group_stop_count++;

                for_other_threads(current, t) {
                        /*
                         * Setting state to TASK_STOPPED for a group
                         * stop is always done with the siglock held,
                         * so this check has no races.
                         */
                        if (!task_is_stopped(t) &&
                            task_set_jobctl_pending(t, signr | gstop)) {
                                sig->group_stop_count++;
                                if (likely(!(t->ptrace & PT_SEIZED)))
                                        signal_wake_up(t, 0);
                                else
                                        ptrace_trap_notify(t);
                        }
                }
        }

        if (likely(!current->ptrace)) {
                int notify = 0;

                /*
                 * If there are no other threads in the group, or if there
                 * is a group stop in progress and we are the last to stop,
                 * report to the parent.
                 */
                if (task_participate_group_stop(current))
                        notify = CLD_STOPPED;

                current->jobctl |= JOBCTL_STOPPED;
                set_special_state(TASK_STOPPED);
                spin_unlock_irq(&current->sighand->siglock);

                /*
                 * Notify the parent of the group stop completion.  Because
                 * we're not holding either the siglock or tasklist_lock
                 * here, ptracer may attach inbetween; however, this is for
                 * group stop and should always be delivered to the real
                 * parent of the group leader.  The new ptracer will get
                 * its notification when this task transitions into
                 * TASK_TRACED.
                 */
                if (notify) {
                        read_lock(&tasklist_lock);
                        do_notify_parent_cldstop(current, false, notify);
                        read_unlock(&tasklist_lock);
                }

                /* Now we don't run again until woken by SIGCONT or SIGKILL */
                cgroup_enter_frozen();
                schedule();
                return true;
        } else {
                /*
                 * While ptraced, group stop is handled by STOP trap.
                 * Schedule it and let the caller deal with it.
                 */
                task_set_jobctl_pending(current, JOBCTL_TRAP_STOP);
                return false;
        }
}

/**
 * do_jobctl_trap - take care of ptrace jobctl traps
 *
 * When PT_SEIZED, it's used for both group stop and explicit
 * SEIZE/INTERRUPT traps.  Both generate PTRACE_EVENT_STOP trap with
 * accompanying siginfo.  If stopped, lower eight bits of exit_code contain
 * the stop signal; otherwise, %SIGTRAP.
 *
 * When !PT_SEIZED, it's used only for group stop trap with stop signal
 * number as exit_code and no siginfo.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held, which may be
 * released and re-acquired before returning with intervening sleep.
 */
static void do_jobctl_trap(void)
{
        struct signal_struct *signal = current->signal;
        int signr = current->jobctl & JOBCTL_STOP_SIGMASK;

        if (current->ptrace & PT_SEIZED) {
                if (!signal->group_stop_count &&
                    !(signal->flags & SIGNAL_STOP_STOPPED))
                        signr = SIGTRAP;
                WARN_ON_ONCE(!signr);
                ptrace_do_notify(signr, signr | (PTRACE_EVENT_STOP << 8),
                                 CLD_STOPPED, 0);
        } else {
                WARN_ON_ONCE(!signr);
                ptrace_stop(signr, CLD_STOPPED, 0, NULL);
        }
}

/**
 * do_freezer_trap - handle the freezer jobctl trap
 *
 * Puts the task into frozen state, if only the task is not about to quit.
 * In this case it drops JOBCTL_TRAP_FREEZE.
 *
 * CONTEXT:
 * Must be called with @current->sighand->siglock held,
 * which is always released before returning.
 */
static void do_freezer_trap(void)
        __releases(&current->sighand->siglock)
{
        /*
         * If there are other trap bits pending except JOBCTL_TRAP_FREEZE,
         * let's make another loop to give it a chance to be handled.
         * In any case, we'll return back.
         */
        if ((current->jobctl & (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE)) !=
             JOBCTL_TRAP_FREEZE) {
                spin_unlock_irq(&current->sighand->siglock);
                return;
        }

        /*
         * Now we're sure that there is no pending fatal signal and no
         * pending traps. Clear TIF_SIGPENDING to not get out of schedule()
         * immediately (if there is a non-fatal signal pending), and
         * put the task into sleep.
         */
        __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
        clear_thread_flag(TIF_SIGPENDING);
        spin_unlock_irq(&current->sighand->siglock);
        cgroup_enter_frozen();
        schedule();

        /*
         * We could've been woken by task_work, run it to clear
         * TIF_NOTIFY_SIGNAL. The caller will retry if necessary.
         */
        clear_notify_signal();
        if (unlikely(task_work_pending(current)))
                task_work_run();
}

static int ptrace_signal(int signr, kernel_siginfo_t *info, enum pid_type type)
{
        /*
         * We do not check sig_kernel_stop(signr) but set this marker
         * unconditionally because we do not know whether debugger will
         * change signr. This flag has no meaning unless we are going
         * to stop after return from ptrace_stop(). In this case it will
         * be checked in do_signal_stop(), we should only stop if it was
         * not cleared by SIGCONT while we were sleeping. See also the
         * comment in dequeue_signal().
         */
        current->jobctl |= JOBCTL_STOP_DEQUEUED;
        signr = ptrace_stop(signr, CLD_TRAPPED, 0, info);

        /* We're back.  Did the debugger cancel the sig?  */
        if (signr == 0)
                return signr;

        /*
         * Update the siginfo structure if the signal has
         * changed.  If the debugger wanted something
         * specific in the siginfo structure then it should
         * have updated *info via PTRACE_SETSIGINFO.
         */
        if (signr != info->si_signo) {
                clear_siginfo(info);
                info->si_signo = signr;
                info->si_errno = 0;
                info->si_code = SI_USER;
                rcu_read_lock();
                info->si_pid = task_pid_vnr(current->parent);
                info->si_uid = from_kuid_munged(current_user_ns(),
                                                task_uid(current->parent));
                rcu_read_unlock();
        }

        /* If the (new) signal is now blocked, requeue it.  */
        if (sigismember(&current->blocked, signr) ||
            fatal_signal_pending(current)) {
                send_signal_locked(signr, info, current, type);
                signr = 0;
        }

        return signr;
}

static void hide_si_addr_tag_bits(struct ksignal *ksig)
{
        switch (siginfo_layout(ksig->sig, ksig->info.si_code)) {
        case SIL_FAULT:
        case SIL_FAULT_TRAPNO:
        case SIL_FAULT_MCEERR:
        case SIL_FAULT_BNDERR:
        case SIL_FAULT_PKUERR:
        case SIL_FAULT_PERF_EVENT:
                ksig->info.si_addr = arch_untagged_si_addr(
                        ksig->info.si_addr, ksig->sig, ksig->info.si_code);
                break;
        case SIL_KILL:
        case SIL_TIMER:
        case SIL_POLL:
        case SIL_CHLD:
        case SIL_RT:
        case SIL_SYS:
                break;
        }
}

bool get_signal(struct ksignal *ksig)
{
        struct sighand_struct *sighand = current->sighand;
        struct signal_struct *signal = current->signal;
        int signr;

        clear_notify_signal();
        if (unlikely(task_work_pending(current)))
                task_work_run();

        if (!task_sigpending(current))
                return false;

        if (unlikely(uprobe_deny_signal()))
                return false;

        /*
         * Do this once, we can't return to user-mode if freezing() == T.
         * do_signal_stop() and ptrace_stop() do freezable_schedule() and
         * thus do not need another check after return.
         */
        try_to_freeze();

relock:
        spin_lock_irq(&sighand->siglock);

        /*
         * Every stopped thread goes here after wakeup. Check to see if
         * we should notify the parent, prepare_signal(SIGCONT) encodes
         * the CLD_ si_code into SIGNAL_CLD_MASK bits.
         */
        if (unlikely(signal->flags & SIGNAL_CLD_MASK)) {
                int why;

                if (signal->flags & SIGNAL_CLD_CONTINUED)
                        why = CLD_CONTINUED;
                else
                        why = CLD_STOPPED;

                signal->flags &= ~SIGNAL_CLD_MASK;

                spin_unlock_irq(&sighand->siglock);

                /*
                 * Notify the parent that we're continuing.  This event is
                 * always per-process and doesn't make whole lot of sense
                 * for ptracers, who shouldn't consume the state via
                 * wait(2) either, but, for backward compatibility, notify
                 * the ptracer of the group leader too unless it's gonna be
                 * a duplicate.
                 */
                read_lock(&tasklist_lock);
                do_notify_parent_cldstop(current, false, why);

                if (ptrace_reparented(current->group_leader))
                        do_notify_parent_cldstop(current->group_leader,
                                                true, why);
                read_unlock(&tasklist_lock);

                goto relock;
        }

        for (;;) {
                struct k_sigaction *ka;
                enum pid_type type;

                /* Has this task already been marked for death? */
                if ((signal->flags & SIGNAL_GROUP_EXIT) ||
                     signal->group_exec_task) {
                        signr = SIGKILL;
                        sigdelset(&current->pending.signal, SIGKILL);
                        trace_signal_deliver(SIGKILL, SEND_SIG_NOINFO,
                                             &sighand->action[SIGKILL-1]);
                        recalc_sigpending();
                        /*
                         * implies do_group_exit() or return to PF_USER_WORKER,
                         * no need to initialize ksig->info/etc.
                         */
                        goto fatal;
                }

                if (unlikely(current->jobctl & JOBCTL_STOP_PENDING) &&
                    do_signal_stop(0))
                        goto relock;

                if (unlikely(current->jobctl &
                             (JOBCTL_TRAP_MASK | JOBCTL_TRAP_FREEZE))) {
                        if (current->jobctl & JOBCTL_TRAP_MASK) {
                                do_jobctl_trap();
                                spin_unlock_irq(&sighand->siglock);
                        } else if (current->jobctl & JOBCTL_TRAP_FREEZE)
                                do_freezer_trap();

                        goto relock;
                }

                /*
                 * If the task is leaving the frozen state, let's update
                 * cgroup counters and reset the frozen bit.
                 */
                if (unlikely(cgroup_task_frozen(current))) {
                        spin_unlock_irq(&sighand->siglock);
                        cgroup_leave_frozen(false);
                        goto relock;
                }

                /*
                 * Signals generated by the execution of an instruction
                 * need to be delivered before any other pending signals
                 * so that the instruction pointer in the signal stack
                 * frame points to the faulting instruction.
                 */
                type = PIDTYPE_PID;
                signr = dequeue_synchronous_signal(&ksig->info);
                if (!signr)
                        signr = dequeue_signal(&current->blocked, &ksig->info, &type);

                if (!signr)
                        break; /* will return 0 */

                if (unlikely(current->ptrace) && (signr != SIGKILL) &&
                    !(sighand->action[signr -1].sa.sa_flags & SA_IMMUTABLE)) {
                        signr = ptrace_signal(signr, &ksig->info, type);
                        if (!signr)
                                continue;
                }

                ka = &sighand->action[signr-1];

                /* Trace actually delivered signals. */
                trace_signal_deliver(signr, &ksig->info, ka);

                if (ka->sa.sa_handler == SIG_IGN) /* Do nothing.  */
                        continue;
                if (ka->sa.sa_handler != SIG_DFL) {
                        /* Run the handler.  */
                        ksig->ka = *ka;

                        if (ka->sa.sa_flags & SA_ONESHOT)
                                ka->sa.sa_handler = SIG_DFL;

                        break; /* will return non-zero "signr" value */
                }

                /*
                 * Now we are doing the default action for this signal.
                 */
                if (sig_kernel_ignore(signr)) /* Default is nothing. */
                        continue;

                /*
                 * Global init gets no signals it doesn't want.
                 * Container-init gets no signals it doesn't want from same
                 * container.
                 *
                 * Note that if global/container-init sees a sig_kernel_only()
                 * signal here, the signal must have been generated internally
                 * or must have come from an ancestor namespace. In either
                 * case, the signal cannot be dropped.
                 */
                if (unlikely(signal->flags & SIGNAL_UNKILLABLE) &&
                                !sig_kernel_only(signr))
                        continue;

                if (sig_kernel_stop(signr)) {
                        /*
                         * The default action is to stop all threads in
                         * the thread group.  The job control signals
                         * do nothing in an orphaned pgrp, but SIGSTOP
                         * always works.  Note that siglock needs to be
                         * dropped during the call to is_orphaned_pgrp()
                         * because of lock ordering with tasklist_lock.
                         * This allows an intervening SIGCONT to be posted.
                         * We need to check for that and bail out if necessary.
                         */
                        if (signr != SIGSTOP) {
                                spin_unlock_irq(&sighand->siglock);

                                /* signals can be posted during this window */

                                if (is_current_pgrp_orphaned())
                                        goto relock;

                                spin_lock_irq(&sighand->siglock);
                        }

                        if (likely(do_signal_stop(signr))) {
                                /* It released the siglock.  */
                                goto relock;
                        }

                        /*
                         * We didn't actually stop, due to a race
                         * with SIGCONT or something like that.
                         */
                        continue;
                }

        fatal:
                spin_unlock_irq(&sighand->siglock);
                if (unlikely(cgroup_task_frozen(current)))
                        cgroup_leave_frozen(true);

                /*
                 * Anything else is fatal, maybe with a core dump.
                 */
                current->flags |= PF_SIGNALED;

                if (sig_kernel_coredump(signr)) {
                        if (print_fatal_signals)
                                print_fatal_signal(signr);
                        proc_coredump_connector(current);
                        /*
                         * If it was able to dump core, this kills all
                         * other threads in the group and synchronizes with
                         * their demise.  If we lost the race with another
                         * thread getting here, it set group_exit_code
                         * first and our do_group_exit call below will use
                         * that value and ignore the one we pass it.
                         */
                        vfs_coredump(&ksig->info);
                }

                /*
                 * PF_USER_WORKER threads will catch and exit on fatal signals
                 * themselves. They have cleanup that must be performed, so we
                 * cannot call do_exit() on their behalf. Note that ksig won't
                 * be properly initialized, PF_USER_WORKER's shouldn't use it.
                 */
                if (current->flags & PF_USER_WORKER)
                        goto out;

                /*
                 * Death signals, no core dump.
                 */
                do_group_exit(signr);
                /* NOTREACHED */
        }
        spin_unlock_irq(&sighand->siglock);

        ksig->sig = signr;

        if (signr && !(ksig->ka.sa.sa_flags & SA_EXPOSE_TAGBITS))
                hide_si_addr_tag_bits(ksig);
out:
        return signr > 0;
}

/**
 * signal_delivered - called after signal delivery to update blocked signals
 * @ksig:                kernel signal struct
 * @stepping:                nonzero if debugger single-step or block-step in use
 *
 * This function should be called when a signal has successfully been
 * delivered. It updates the blocked signals accordingly (@ksig->ka.sa.sa_mask
 * is always blocked), and the signal itself is blocked unless %SA_NODEFER
 * is set in @ksig->ka.sa.sa_flags.  Tracing is notified.
 */
static void signal_delivered(struct ksignal *ksig, int stepping)
{
        sigset_t blocked;

        /* A signal was successfully delivered, and the
           saved sigmask was stored on the signal frame,
           and will be restored by sigreturn.  So we can
           simply clear the restore sigmask flag.  */
        clear_restore_sigmask();

        sigorsets(&blocked, &current->blocked, &ksig->ka.sa.sa_mask);
        if (!(ksig->ka.sa.sa_flags & SA_NODEFER))
                sigaddset(&blocked, ksig->sig);
        set_current_blocked(&blocked);
        if (current->sas_ss_flags & SS_AUTODISARM)
                sas_ss_reset(current);
        if (stepping)
                ptrace_notify(SIGTRAP, 0);
}

void signal_setup_done(int failed, struct ksignal *ksig, int stepping)
{
        if (failed)
                force_sigsegv(ksig->sig);
        else
                signal_delivered(ksig, stepping);
}

/*
 * It could be that complete_signal() picked us to notify about the
 * group-wide signal. Other threads should be notified now to take
 * the shared signals in @which since we will not.
 */
static void retarget_shared_pending(struct task_struct *tsk, sigset_t *which)
{
        sigset_t retarget;
        struct task_struct *t;

        sigandsets(&retarget, &tsk->signal->shared_pending.signal, which);
        if (sigisemptyset(&retarget))
                return;

        for_other_threads(tsk, t) {
                if (t->flags & PF_EXITING)
                        continue;

                if (!has_pending_signals(&retarget, &t->blocked))
                        continue;
                /* Remove the signals this thread can handle. */
                sigandsets(&retarget, &retarget, &t->blocked);

                if (!task_sigpending(t))
                        signal_wake_up(t, 0);

                if (sigisemptyset(&retarget))
                        break;
        }
}

void exit_signals(struct task_struct *tsk)
{
        int group_stop = 0;
        sigset_t unblocked;

        /*
         * @tsk is about to have PF_EXITING set - lock out users which
         * expect stable threadgroup.
         */
        cgroup_threadgroup_change_begin(tsk);

        if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
                sched_mm_cid_exit_signals(tsk);
                tsk->flags |= PF_EXITING;
                cgroup_threadgroup_change_end(tsk);
                return;
        }

        spin_lock_irq(&tsk->sighand->siglock);
        /*
         * From now this task is not visible for group-wide signals,
         * see wants_signal(), do_signal_stop().
         */
        sched_mm_cid_exit_signals(tsk);
        tsk->flags |= PF_EXITING;

        cgroup_threadgroup_change_end(tsk);

        if (!task_sigpending(tsk))
                goto out;

        unblocked = tsk->blocked;
        signotset(&unblocked);
        retarget_shared_pending(tsk, &unblocked);

        if (unlikely(tsk->jobctl & JOBCTL_STOP_PENDING) &&
            task_participate_group_stop(tsk))
                group_stop = CLD_STOPPED;
out:
        spin_unlock_irq(&tsk->sighand->siglock);

        /*
         * If group stop has completed, deliver the notification.  This
         * should always go to the real parent of the group leader.
         */
        if (unlikely(group_stop)) {
                read_lock(&tasklist_lock);
                do_notify_parent_cldstop(tsk, false, group_stop);
                read_unlock(&tasklist_lock);
        }
}

/*
 * System call entry points.
 */

/**
 *  sys_restart_syscall - restart a system call
 */
SYSCALL_DEFINE0(restart_syscall)
{
        struct restart_block *restart = &current->restart_block;
        return restart->fn(restart);
}

long do_no_restart_syscall(struct restart_block *param)
{
        return -EINTR;
}

static void __set_task_blocked(struct task_struct *tsk, const sigset_t *newset)
{
        if (task_sigpending(tsk) && !thread_group_empty(tsk)) {
                sigset_t newblocked;
                /* A set of now blocked but previously unblocked signals. */
                sigandnsets(&newblocked, newset, &current->blocked);
                retarget_shared_pending(tsk, &newblocked);
        }
        tsk->blocked = *newset;
        recalc_sigpending();
}

/**
 * set_current_blocked - change current->blocked mask
 * @newset: new mask
 *
 * It is wrong to change ->blocked directly, this helper should be used
 * to ensure the process can't miss a shared signal we are going to block.
 */
void set_current_blocked(sigset_t *newset)
{
        sigdelsetmask(newset, sigmask(SIGKILL) | sigmask(SIGSTOP));
        __set_current_blocked(newset);
}

void __set_current_blocked(const sigset_t *newset)
{
        struct task_struct *tsk = current;

        /*
         * In case the signal mask hasn't changed, there is nothing we need
         * to do. The current->blocked shouldn't be modified by other task.
         */
        if (sigequalsets(&tsk->blocked, newset))
                return;

        spin_lock_irq(&tsk->sighand->siglock);
        __set_task_blocked(tsk, newset);
        spin_unlock_irq(&tsk->sighand->siglock);
}

/*
 * This is also useful for kernel threads that want to temporarily
 * (or permanently) block certain signals.
 *
 * NOTE! Unlike the user-mode sys_sigprocmask(), the kernel
 * interface happily blocks "unblockable" signals like SIGKILL
 * and friends.
 */
int sigprocmask(int how, sigset_t *set, sigset_t *oldset)
{
        struct task_struct *tsk = current;
        sigset_t newset;

        /* Lockless, only current can change ->blocked, never from irq */
        if (oldset)
                *oldset = tsk->blocked;

        switch (how) {
        case SIG_BLOCK:
                sigorsets(&newset, &tsk->blocked, set);
                break;
        case SIG_UNBLOCK:
                sigandnsets(&newset, &tsk->blocked, set);
                break;
        case SIG_SETMASK:
                newset = *set;
                break;
        default:
                return -EINVAL;
        }

        __set_current_blocked(&newset);
        return 0;
}
EXPORT_SYMBOL(sigprocmask);

/*
 * The api helps set app-provided sigmasks.
 *
 * This is useful for syscalls such as ppoll, pselect, io_pgetevents and
 * epoll_pwait where a new sigmask is passed from userland for the syscalls.
 *
 * Note that it does set_restore_sigmask() in advance, so it must be always
 * paired with restore_saved_sigmask_unless() before return from syscall.
 */
int set_user_sigmask(const sigset_t __user *umask, size_t sigsetsize)
{
        sigset_t kmask;

        if (!umask)
                return 0;
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;
        if (copy_from_user(&kmask, umask, sizeof(sigset_t)))
                return -EFAULT;

        set_restore_sigmask();
        current->saved_sigmask = current->blocked;
        set_current_blocked(&kmask);

        return 0;
}

#ifdef CONFIG_COMPAT
int set_compat_user_sigmask(const compat_sigset_t __user *umask,
                            size_t sigsetsize)
{
        sigset_t kmask;

        if (!umask)
                return 0;
        if (sigsetsize != sizeof(compat_sigset_t))
                return -EINVAL;
        if (get_compat_sigset(&kmask, umask))
                return -EFAULT;

        set_restore_sigmask();
        current->saved_sigmask = current->blocked;
        set_current_blocked(&kmask);

        return 0;
}
#endif

/**
 *  sys_rt_sigprocmask - change the list of currently blocked signals
 *  @how: whether to add, remove, or set signals
 *  @nset: stores pending signals
 *  @oset: previous value of signal mask if non-null
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigprocmask, int, how, sigset_t __user *, nset,
                sigset_t __user *, oset, size_t, sigsetsize)
{
        sigset_t old_set, new_set;
        int error;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        old_set = current->blocked;

        if (nset) {
                if (copy_from_user(&new_set, nset, sizeof(sigset_t)))
                        return -EFAULT;
                sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));

                error = sigprocmask(how, &new_set, NULL);
                if (error)
                        return error;
        }

        if (oset) {
                if (copy_to_user(oset, &old_set, sizeof(sigset_t)))
                        return -EFAULT;
        }

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigprocmask, int, how, compat_sigset_t __user *, nset,
                compat_sigset_t __user *, oset, compat_size_t, sigsetsize)
{
        sigset_t old_set = current->blocked;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (nset) {
                sigset_t new_set;
                int error;
                if (get_compat_sigset(&new_set, nset))
                        return -EFAULT;
                sigdelsetmask(&new_set, sigmask(SIGKILL)|sigmask(SIGSTOP));

                error = sigprocmask(how, &new_set, NULL);
                if (error)
                        return error;
        }
        return oset ? put_compat_sigset(oset, &old_set, sizeof(*oset)) : 0;
}
#endif

static void do_sigpending(sigset_t *set)
{
        spin_lock_irq(&current->sighand->siglock);
        sigorsets(set, &current->pending.signal,
                  &current->signal->shared_pending.signal);
        spin_unlock_irq(&current->sighand->siglock);

        /* Outside the lock because only this thread touches it.  */
        sigandsets(set, &current->blocked, set);
}

/**
 *  sys_rt_sigpending - examine a pending signal that has been raised
 *                        while blocked
 *  @uset: stores pending signals
 *  @sigsetsize: size of sigset_t type or larger
 */
SYSCALL_DEFINE2(rt_sigpending, sigset_t __user *, uset, size_t, sigsetsize)
{
        sigset_t set;

        if (sigsetsize > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        if (copy_to_user(uset, &set, sigsetsize))
                return -EFAULT;

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigpending, compat_sigset_t __user *, uset,
                compat_size_t, sigsetsize)
{
        sigset_t set;

        if (sigsetsize > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        return put_compat_sigset(uset, &set, sigsetsize);
}
#endif

static const struct {
        unsigned char limit, layout;
} sig_sicodes[] = {
        [SIGILL]  = { NSIGILL,  SIL_FAULT },
        [SIGFPE]  = { NSIGFPE,  SIL_FAULT },
        [SIGSEGV] = { NSIGSEGV, SIL_FAULT },
        [SIGBUS]  = { NSIGBUS,  SIL_FAULT },
        [SIGTRAP] = { NSIGTRAP, SIL_FAULT },
#if defined(SIGEMT)
        [SIGEMT]  = { NSIGEMT,  SIL_FAULT },
#endif
        [SIGCHLD] = { NSIGCHLD, SIL_CHLD },
        [SIGPOLL] = { NSIGPOLL, SIL_POLL },
        [SIGSYS]  = { NSIGSYS,  SIL_SYS },
};

static bool known_siginfo_layout(unsigned sig, int si_code)
{
        if (si_code == SI_KERNEL)
                return true;
        else if ((si_code > SI_USER)) {
                if (sig_specific_sicodes(sig)) {
                        if (si_code <= sig_sicodes[sig].limit)
                                return true;
                }
                else if (si_code <= NSIGPOLL)
                        return true;
        }
        else if (si_code >= SI_DETHREAD)
                return true;
        else if (si_code == SI_ASYNCNL)
                return true;
        return false;
}

enum siginfo_layout siginfo_layout(unsigned sig, int si_code)
{
        enum siginfo_layout layout = SIL_KILL;
        if ((si_code > SI_USER) && (si_code < SI_KERNEL)) {
                if ((sig < ARRAY_SIZE(sig_sicodes)) &&
                    (si_code <= sig_sicodes[sig].limit)) {
                        layout = sig_sicodes[sig].layout;
                        /* Handle the exceptions */
                        if ((sig == SIGBUS) &&
                            (si_code >= BUS_MCEERR_AR) && (si_code <= BUS_MCEERR_AO))
                                layout = SIL_FAULT_MCEERR;
                        else if ((sig == SIGSEGV) && (si_code == SEGV_BNDERR))
                                layout = SIL_FAULT_BNDERR;
#ifdef SEGV_PKUERR
                        else if ((sig == SIGSEGV) && (si_code == SEGV_PKUERR))
                                layout = SIL_FAULT_PKUERR;
#endif
                        else if ((sig == SIGTRAP) && (si_code == TRAP_PERF))
                                layout = SIL_FAULT_PERF_EVENT;
                        else if (IS_ENABLED(CONFIG_SPARC) &&
                                 (sig == SIGILL) && (si_code == ILL_ILLTRP))
                                layout = SIL_FAULT_TRAPNO;
                        else if (IS_ENABLED(CONFIG_ALPHA) &&
                                 ((sig == SIGFPE) ||
                                  ((sig == SIGTRAP) && (si_code == TRAP_UNK))))
                                layout = SIL_FAULT_TRAPNO;
                }
                else if (si_code <= NSIGPOLL)
                        layout = SIL_POLL;
        } else {
                if (si_code == SI_TIMER)
                        layout = SIL_TIMER;
                else if (si_code == SI_SIGIO)
                        layout = SIL_POLL;
                else if (si_code < 0)
                        layout = SIL_RT;
        }
        return layout;
}

static inline char __user *si_expansion(const siginfo_t __user *info)
{
        return ((char __user *)info) + sizeof(struct kernel_siginfo);
}

int copy_siginfo_to_user(siginfo_t __user *to, const kernel_siginfo_t *from)
{
        char __user *expansion = si_expansion(to);
        if (copy_to_user(to, from , sizeof(struct kernel_siginfo)))
                return -EFAULT;
        if (clear_user(expansion, SI_EXPANSION_SIZE))
                return -EFAULT;
        return 0;
}

static int post_copy_siginfo_from_user(kernel_siginfo_t *info,
                                       const siginfo_t __user *from)
{
        if (unlikely(!known_siginfo_layout(info->si_signo, info->si_code))) {
                char __user *expansion = si_expansion(from);
                char buf[SI_EXPANSION_SIZE];
                int i;
                /*
                 * An unknown si_code might need more than
                 * sizeof(struct kernel_siginfo) bytes.  Verify all of the
                 * extra bytes are 0.  This guarantees copy_siginfo_to_user
                 * will return this data to userspace exactly.
                 */
                if (copy_from_user(&buf, expansion, SI_EXPANSION_SIZE))
                        return -EFAULT;
                for (i = 0; i < SI_EXPANSION_SIZE; i++) {
                        if (buf[i] != 0)
                                return -E2BIG;
                }
        }
        return 0;
}

static int __copy_siginfo_from_user(int signo, kernel_siginfo_t *to,
                                    const siginfo_t __user *from)
{
        if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
                return -EFAULT;
        to->si_signo = signo;
        return post_copy_siginfo_from_user(to, from);
}

int copy_siginfo_from_user(kernel_siginfo_t *to, const siginfo_t __user *from)
{
        if (copy_from_user(to, from, sizeof(struct kernel_siginfo)))
                return -EFAULT;
        return post_copy_siginfo_from_user(to, from);
}

#ifdef CONFIG_COMPAT
/**
 * copy_siginfo_to_external32 - copy a kernel siginfo into a compat user siginfo
 * @to: compat siginfo destination
 * @from: kernel siginfo source
 *
 * Note: This function does not work properly for the SIGCHLD on x32, but
 * fortunately it doesn't have to.  The only valid callers for this function are
 * copy_siginfo_to_user32, which is overriden for x32 and the coredump code.
 * The latter does not care because SIGCHLD will never cause a coredump.
 */
void copy_siginfo_to_external32(struct compat_siginfo *to,
                const struct kernel_siginfo *from)
{
        memset(to, 0, sizeof(*to));

        to->si_signo = from->si_signo;
        to->si_errno = from->si_errno;
        to->si_code  = from->si_code;
        switch(siginfo_layout(from->si_signo, from->si_code)) {
        case SIL_KILL:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                break;
        case SIL_TIMER:
                to->si_tid     = from->si_tid;
                to->si_overrun = from->si_overrun;
                to->si_int     = from->si_int;
                break;
        case SIL_POLL:
                to->si_band = from->si_band;
                to->si_fd   = from->si_fd;
                break;
        case SIL_FAULT:
                to->si_addr = ptr_to_compat(from->si_addr);
                break;
        case SIL_FAULT_TRAPNO:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_trapno = from->si_trapno;
                break;
        case SIL_FAULT_MCEERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_addr_lsb = from->si_addr_lsb;
                break;
        case SIL_FAULT_BNDERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_lower = ptr_to_compat(from->si_lower);
                to->si_upper = ptr_to_compat(from->si_upper);
                break;
        case SIL_FAULT_PKUERR:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_pkey = from->si_pkey;
                break;
        case SIL_FAULT_PERF_EVENT:
                to->si_addr = ptr_to_compat(from->si_addr);
                to->si_perf_data = from->si_perf_data;
                to->si_perf_type = from->si_perf_type;
                to->si_perf_flags = from->si_perf_flags;
                break;
        case SIL_CHLD:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_status = from->si_status;
                to->si_utime = from->si_utime;
                to->si_stime = from->si_stime;
                break;
        case SIL_RT:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_int = from->si_int;
                break;
        case SIL_SYS:
                to->si_call_addr = ptr_to_compat(from->si_call_addr);
                to->si_syscall   = from->si_syscall;
                to->si_arch      = from->si_arch;
                break;
        }
}

int __copy_siginfo_to_user32(struct compat_siginfo __user *to,
                           const struct kernel_siginfo *from)
{
        struct compat_siginfo new;

        copy_siginfo_to_external32(&new, from);
        if (copy_to_user(to, &new, sizeof(struct compat_siginfo)))
                return -EFAULT;
        return 0;
}

static int post_copy_siginfo_from_user32(kernel_siginfo_t *to,
                                         const struct compat_siginfo *from)
{
        clear_siginfo(to);
        to->si_signo = from->si_signo;
        to->si_errno = from->si_errno;
        to->si_code  = from->si_code;
        switch(siginfo_layout(from->si_signo, from->si_code)) {
        case SIL_KILL:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                break;
        case SIL_TIMER:
                to->si_tid     = from->si_tid;
                to->si_overrun = from->si_overrun;
                to->si_int     = from->si_int;
                break;
        case SIL_POLL:
                to->si_band = from->si_band;
                to->si_fd   = from->si_fd;
                break;
        case SIL_FAULT:
                to->si_addr = compat_ptr(from->si_addr);
                break;
        case SIL_FAULT_TRAPNO:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_trapno = from->si_trapno;
                break;
        case SIL_FAULT_MCEERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_addr_lsb = from->si_addr_lsb;
                break;
        case SIL_FAULT_BNDERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_lower = compat_ptr(from->si_lower);
                to->si_upper = compat_ptr(from->si_upper);
                break;
        case SIL_FAULT_PKUERR:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_pkey = from->si_pkey;
                break;
        case SIL_FAULT_PERF_EVENT:
                to->si_addr = compat_ptr(from->si_addr);
                to->si_perf_data = from->si_perf_data;
                to->si_perf_type = from->si_perf_type;
                to->si_perf_flags = from->si_perf_flags;
                break;
        case SIL_CHLD:
                to->si_pid    = from->si_pid;
                to->si_uid    = from->si_uid;
                to->si_status = from->si_status;
#ifdef CONFIG_X86_X32_ABI
                if (in_x32_syscall()) {
                        to->si_utime = from->_sifields._sigchld_x32._utime;
                        to->si_stime = from->_sifields._sigchld_x32._stime;
                } else
#endif
                {
                        to->si_utime = from->si_utime;
                        to->si_stime = from->si_stime;
                }
                break;
        case SIL_RT:
                to->si_pid = from->si_pid;
                to->si_uid = from->si_uid;
                to->si_int = from->si_int;
                break;
        case SIL_SYS:
                to->si_call_addr = compat_ptr(from->si_call_addr);
                to->si_syscall   = from->si_syscall;
                to->si_arch      = from->si_arch;
                break;
        }
        return 0;
}

static int __copy_siginfo_from_user32(int signo, struct kernel_siginfo *to,
                                      const struct compat_siginfo __user *ufrom)
{
        struct compat_siginfo from;

        if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
                return -EFAULT;

        from.si_signo = signo;
        return post_copy_siginfo_from_user32(to, &from);
}

int copy_siginfo_from_user32(struct kernel_siginfo *to,
                             const struct compat_siginfo __user *ufrom)
{
        struct compat_siginfo from;

        if (copy_from_user(&from, ufrom, sizeof(struct compat_siginfo)))
                return -EFAULT;

        return post_copy_siginfo_from_user32(to, &from);
}
#endif /* CONFIG_COMPAT */

/**
 *  do_sigtimedwait - wait for queued signals specified in @which
 *  @which: queued signals to wait for
 *  @info: if non-null, the signal's siginfo is returned here
 *  @ts: upper bound on process time suspension
 */
static int do_sigtimedwait(const sigset_t *which, kernel_siginfo_t *info,
                    const struct timespec64 *ts)
{
        ktime_t *to = NULL, timeout = KTIME_MAX;
        struct task_struct *tsk = current;
        sigset_t mask = *which;
        enum pid_type type;
        int sig, ret = 0;

        if (ts) {
                if (!timespec64_valid(ts))
                        return -EINVAL;
                timeout = timespec64_to_ktime(*ts);
                to = &timeout;
        }

        /*
         * Invert the set of allowed signals to get those we want to block.
         */
        sigdelsetmask(&mask, sigmask(SIGKILL) | sigmask(SIGSTOP));
        signotset(&mask);

        spin_lock_irq(&tsk->sighand->siglock);
        sig = dequeue_signal(&mask, info, &type);
        if (!sig && timeout) {
                /*
                 * None ready, temporarily unblock those we're interested
                 * while we are sleeping in so that we'll be awakened when
                 * they arrive. Unblocking is always fine, we can avoid
                 * set_current_blocked().
                 */
                tsk->real_blocked = tsk->blocked;
                sigandsets(&tsk->blocked, &tsk->blocked, &mask);
                recalc_sigpending();
                spin_unlock_irq(&tsk->sighand->siglock);

                __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
                ret = schedule_hrtimeout_range(to, tsk->timer_slack_ns,
                                               HRTIMER_MODE_REL);
                spin_lock_irq(&tsk->sighand->siglock);
                __set_task_blocked(tsk, &tsk->real_blocked);
                sigemptyset(&tsk->real_blocked);
                sig = dequeue_signal(&mask, info, &type);
        }
        spin_unlock_irq(&tsk->sighand->siglock);

        if (sig)
                return sig;
        return ret ? -EINTR : -EAGAIN;
}

/**
 *  sys_rt_sigtimedwait - synchronously wait for queued signals specified
 *                        in @uthese
 *  @uthese: queued signals to wait for
 *  @uinfo: if non-null, the signal's siginfo is returned here
 *  @uts: upper bound on process time suspension
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigtimedwait, const sigset_t __user *, uthese,
                siginfo_t __user *, uinfo,
                const struct __kernel_timespec __user *, uts,
                size_t, sigsetsize)
{
        sigset_t these;
        struct timespec64 ts;
        kernel_siginfo_t info;
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&these, uthese, sizeof(these)))
                return -EFAULT;

        if (uts) {
                if (get_timespec64(&ts, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}

#ifdef CONFIG_COMPAT_32BIT_TIME
SYSCALL_DEFINE4(rt_sigtimedwait_time32, const sigset_t __user *, uthese,
                siginfo_t __user *, uinfo,
                const struct old_timespec32 __user *, uts,
                size_t, sigsetsize)
{
        sigset_t these;
        struct timespec64 ts;
        kernel_siginfo_t info;
        int ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&these, uthese, sizeof(these)))
                return -EFAULT;

        if (uts) {
                if (get_old_timespec32(&ts, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&these, &info, uts ? &ts : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}
#endif

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time64, compat_sigset_t __user *, uthese,
                struct compat_siginfo __user *, uinfo,
                struct __kernel_timespec __user *, uts, compat_size_t, sigsetsize)
{
        sigset_t s;
        struct timespec64 t;
        kernel_siginfo_t info;
        long ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&s, uthese))
                return -EFAULT;

        if (uts) {
                if (get_timespec64(&t, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user32(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}

#ifdef CONFIG_COMPAT_32BIT_TIME
COMPAT_SYSCALL_DEFINE4(rt_sigtimedwait_time32, compat_sigset_t __user *, uthese,
                struct compat_siginfo __user *, uinfo,
                struct old_timespec32 __user *, uts, compat_size_t, sigsetsize)
{
        sigset_t s;
        struct timespec64 t;
        kernel_siginfo_t info;
        long ret;

        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&s, uthese))
                return -EFAULT;

        if (uts) {
                if (get_old_timespec32(&t, uts))
                        return -EFAULT;
        }

        ret = do_sigtimedwait(&s, &info, uts ? &t : NULL);

        if (ret > 0 && uinfo) {
                if (copy_siginfo_to_user32(uinfo, &info))
                        ret = -EFAULT;
        }

        return ret;
}
#endif
#endif

static void prepare_kill_siginfo(int sig, struct kernel_siginfo *info,
                                 enum pid_type type)
{
        clear_siginfo(info);
        info->si_signo = sig;
        info->si_errno = 0;
        info->si_code = (type == PIDTYPE_PID) ? SI_TKILL : SI_USER;
        info->si_pid = task_tgid_vnr(current);
        info->si_uid = from_kuid_munged(current_user_ns(), current_uid());
}

/**
 *  sys_kill - send a signal to a process
 *  @pid: the PID of the process
 *  @sig: signal to be sent
 */
SYSCALL_DEFINE2(kill, pid_t, pid, int, sig)
{
        struct kernel_siginfo info;

        prepare_kill_siginfo(sig, &info, PIDTYPE_TGID);

        return kill_something_info(sig, &info, pid);
}

/*
 * Verify that the signaler and signalee either are in the same pid namespace
 * or that the signaler's pid namespace is an ancestor of the signalee's pid
 * namespace.
 */
static bool access_pidfd_pidns(struct pid *pid)
{
        struct pid_namespace *active = task_active_pid_ns(current);
        struct pid_namespace *p = ns_of_pid(pid);

        for (;;) {
                if (!p)
                        return false;
                if (p == active)
                        break;
                p = p->parent;
        }

        return true;
}

static int copy_siginfo_from_user_any(kernel_siginfo_t *kinfo,
                siginfo_t __user *info)
{
#ifdef CONFIG_COMPAT
        /*
         * Avoid hooking up compat syscalls and instead handle necessary
         * conversions here. Note, this is a stop-gap measure and should not be
         * considered a generic solution.
         */
        if (in_compat_syscall())
                return copy_siginfo_from_user32(
                        kinfo, (struct compat_siginfo __user *)info);
#endif
        return copy_siginfo_from_user(kinfo, info);
}

static struct pid *pidfd_to_pid(const struct file *file)
{
        struct pid *pid;

        pid = pidfd_pid(file);
        if (!IS_ERR(pid))
                return pid;

        return tgid_pidfd_to_pid(file);
}

#define PIDFD_SEND_SIGNAL_FLAGS                            \
        (PIDFD_SIGNAL_THREAD | PIDFD_SIGNAL_THREAD_GROUP | \
         PIDFD_SIGNAL_PROCESS_GROUP)

static int do_pidfd_send_signal(struct pid *pid, int sig, enum pid_type type,
                                siginfo_t __user *info, unsigned int flags)
{
        kernel_siginfo_t kinfo;

        switch (flags) {
        case PIDFD_SIGNAL_THREAD:
                type = PIDTYPE_PID;
                break;
        case PIDFD_SIGNAL_THREAD_GROUP:
                type = PIDTYPE_TGID;
                break;
        case PIDFD_SIGNAL_PROCESS_GROUP:
                type = PIDTYPE_PGID;
                break;
        }

        if (info) {
                int ret;

                ret = copy_siginfo_from_user_any(&kinfo, info);
                if (unlikely(ret))
                        return ret;

                if (unlikely(sig != kinfo.si_signo))
                        return -EINVAL;

                /* Only allow sending arbitrary signals to yourself. */
                if ((task_pid(current) != pid || type > PIDTYPE_TGID) &&
                    (kinfo.si_code >= 0 || kinfo.si_code == SI_TKILL))
                        return -EPERM;
        } else {
                prepare_kill_siginfo(sig, &kinfo, type);
        }

        if (type == PIDTYPE_PGID)
                return kill_pgrp_info(sig, &kinfo, pid);

        return kill_pid_info_type(sig, &kinfo, pid, type);
}

/**
 * sys_pidfd_send_signal - Signal a process through a pidfd
 * @pidfd:  file descriptor of the process
 * @sig:    signal to send
 * @info:   signal info
 * @flags:  future flags
 *
 * Send the signal to the thread group or to the individual thread depending
 * on PIDFD_THREAD.
 * In the future extension to @flags may be used to override the default scope
 * of @pidfd.
 *
 * Return: 0 on success, negative errno on failure
 */
SYSCALL_DEFINE4(pidfd_send_signal, int, pidfd, int, sig,
                siginfo_t __user *, info, unsigned int, flags)
{
        struct pid *pid;
        enum pid_type type;

        /* Enforce flags be set to 0 until we add an extension. */
        if (flags & ~PIDFD_SEND_SIGNAL_FLAGS)
                return -EINVAL;

        /* Ensure that only a single signal scope determining flag is set. */
        if (hweight32(flags & PIDFD_SEND_SIGNAL_FLAGS) > 1)
                return -EINVAL;

        switch (pidfd) {
        case PIDFD_SELF_THREAD:
                pid = get_task_pid(current, PIDTYPE_PID);
                type = PIDTYPE_PID;
                break;
        case PIDFD_SELF_THREAD_GROUP:
                pid = get_task_pid(current, PIDTYPE_TGID);
                type = PIDTYPE_TGID;
                break;
        default: {
                CLASS(fd, f)(pidfd);
                if (fd_empty(f))
                        return -EBADF;

                /* Is this a pidfd? */
                pid = pidfd_to_pid(fd_file(f));
                if (IS_ERR(pid))
                        return PTR_ERR(pid);

                if (!access_pidfd_pidns(pid))
                        return -EINVAL;

                /* Infer scope from the type of pidfd. */
                if (fd_file(f)->f_flags & PIDFD_THREAD)
                        type = PIDTYPE_PID;
                else
                        type = PIDTYPE_TGID;

                return do_pidfd_send_signal(pid, sig, type, info, flags);
        }
        }

        return do_pidfd_send_signal(pid, sig, type, info, flags);
}

static int
do_send_specific(pid_t tgid, pid_t pid, int sig, struct kernel_siginfo *info)
{
        struct task_struct *p;
        int error = -ESRCH;

        rcu_read_lock();
        p = find_task_by_vpid(pid);
        if (p && (tgid <= 0 || task_tgid_vnr(p) == tgid)) {
                error = check_kill_permission(sig, info, p);
                /*
                 * The null signal is a permissions and process existence
                 * probe.  No signal is actually delivered.
                 */
                if (!error && sig) {
                        error = do_send_sig_info(sig, info, p, PIDTYPE_PID);
                        /*
                         * If lock_task_sighand() failed we pretend the task
                         * dies after receiving the signal. The window is tiny,
                         * and the signal is private anyway.
                         */
                        if (unlikely(error == -ESRCH))
                                error = 0;
                }
        }
        rcu_read_unlock();

        return error;
}

static int do_tkill(pid_t tgid, pid_t pid, int sig)
{
        struct kernel_siginfo info;

        prepare_kill_siginfo(sig, &info, PIDTYPE_PID);

        return do_send_specific(tgid, pid, sig, &info);
}

/**
 *  sys_tgkill - send signal to one specific thread
 *  @tgid: the thread group ID of the thread
 *  @pid: the PID of the thread
 *  @sig: signal to be sent
 *
 *  This syscall also checks the @tgid and returns -ESRCH even if the PID
 *  exists but it's not belonging to the target process anymore. This
 *  method solves the problem of threads exiting and PIDs getting reused.
 */
SYSCALL_DEFINE3(tgkill, pid_t, tgid, pid_t, pid, int, sig)
{
        /* This is only valid for single tasks */
        if (pid <= 0 || tgid <= 0)
                return -EINVAL;

        return do_tkill(tgid, pid, sig);
}

/**
 *  sys_tkill - send signal to one specific task
 *  @pid: the PID of the task
 *  @sig: signal to be sent
 *
 *  Send a signal to only one task, even if it's a CLONE_THREAD task.
 */
SYSCALL_DEFINE2(tkill, pid_t, pid, int, sig)
{
        /* This is only valid for single tasks */
        if (pid <= 0)
                return -EINVAL;

        return do_tkill(0, pid, sig);
}

static int do_rt_sigqueueinfo(pid_t pid, int sig, kernel_siginfo_t *info)
{
        /* Not even root can pretend to send signals from the kernel.
         * Nor can they impersonate a kill()/tgkill(), which adds source info.
         */
        if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
            (task_pid_vnr(current) != pid))
                return -EPERM;

        /* POSIX.1b doesn't mention process groups.  */
        return kill_proc_info(sig, info, pid);
}

/**
 *  sys_rt_sigqueueinfo - send signal information to a signal
 *  @pid: the PID of the thread
 *  @sig: signal to be sent
 *  @uinfo: signal info to be sent
 */
SYSCALL_DEFINE3(rt_sigqueueinfo, pid_t, pid, int, sig,
                siginfo_t __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_sigqueueinfo(pid, sig, &info);
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(rt_sigqueueinfo,
                        compat_pid_t, pid,
                        int, sig,
                        struct compat_siginfo __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_sigqueueinfo(pid, sig, &info);
}
#endif

static int do_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, kernel_siginfo_t *info)
{
        /* This is only valid for single tasks */
        if (pid <= 0 || tgid <= 0)
                return -EINVAL;

        /* Not even root can pretend to send signals from the kernel.
         * Nor can they impersonate a kill()/tgkill(), which adds source info.
         */
        if ((info->si_code >= 0 || info->si_code == SI_TKILL) &&
            (task_pid_vnr(current) != pid))
                return -EPERM;

        return do_send_specific(tgid, pid, sig, info);
}

SYSCALL_DEFINE4(rt_tgsigqueueinfo, pid_t, tgid, pid_t, pid, int, sig,
                siginfo_t __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_tgsigqueueinfo,
                        compat_pid_t, tgid,
                        compat_pid_t, pid,
                        int, sig,
                        struct compat_siginfo __user *, uinfo)
{
        kernel_siginfo_t info;
        int ret = __copy_siginfo_from_user32(sig, &info, uinfo);
        if (unlikely(ret))
                return ret;
        return do_rt_tgsigqueueinfo(tgid, pid, sig, &info);
}
#endif

/*
 * For kthreads only, must not be used if cloned with CLONE_SIGHAND
 */
void kernel_sigaction(int sig, __sighandler_t action)
{
        spin_lock_irq(&current->sighand->siglock);
        current->sighand->action[sig - 1].sa.sa_handler = action;
        if (action == SIG_IGN) {
                sigset_t mask;

                sigemptyset(&mask);
                sigaddset(&mask, sig);

                flush_sigqueue_mask(current, &mask, &current->signal->shared_pending);
                flush_sigqueue_mask(current, &mask, &current->pending);
                recalc_sigpending();
        }
        spin_unlock_irq(&current->sighand->siglock);
}
EXPORT_SYMBOL(kernel_sigaction);

void __weak sigaction_compat_abi(struct k_sigaction *act,
                struct k_sigaction *oact)
{
}

int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
{
        struct task_struct *p = current, *t;
        struct k_sigaction *k;
        sigset_t mask;

        if (!valid_signal(sig) || sig < 1 || (act && sig_kernel_only(sig)))
                return -EINVAL;

        k = &p->sighand->action[sig-1];

        spin_lock_irq(&p->sighand->siglock);
        if (k->sa.sa_flags & SA_IMMUTABLE) {
                spin_unlock_irq(&p->sighand->siglock);
                return -EINVAL;
        }
        if (oact)
                *oact = *k;

        /*
         * Make sure that we never accidentally claim to support SA_UNSUPPORTED,
         * e.g. by having an architecture use the bit in their uapi.
         */
        BUILD_BUG_ON(UAPI_SA_FLAGS & SA_UNSUPPORTED);

        /*
         * Clear unknown flag bits in order to allow userspace to detect missing
         * support for flag bits and to allow the kernel to use non-uapi bits
         * internally.
         */
        if (act)
                act->sa.sa_flags &= UAPI_SA_FLAGS;
        if (oact)
                oact->sa.sa_flags &= UAPI_SA_FLAGS;

        sigaction_compat_abi(act, oact);

        if (act) {
                bool was_ignored = k->sa.sa_handler == SIG_IGN;

                sigdelsetmask(&act->sa.sa_mask,
                              sigmask(SIGKILL) | sigmask(SIGSTOP));
                *k = *act;
                /*
                 * POSIX 3.3.1.3:
                 *  "Setting a signal action to SIG_IGN for a signal that is
                 *   pending shall cause the pending signal to be discarded,
                 *   whether or not it is blocked."
                 *
                 *  "Setting a signal action to SIG_DFL for a signal that is
                 *   pending and whose default action is to ignore the signal
                 *   (for example, SIGCHLD), shall cause the pending signal to
                 *   be discarded, whether or not it is blocked"
                 */
                if (sig_handler_ignored(sig_handler(p, sig), sig)) {
                        sigemptyset(&mask);
                        sigaddset(&mask, sig);
                        flush_sigqueue_mask(p, &mask, &p->signal->shared_pending);
                        for_each_thread(p, t)
                                flush_sigqueue_mask(p, &mask, &t->pending);
                } else if (was_ignored) {
                        posixtimer_sig_unignore(p, sig);
                }
        }

        spin_unlock_irq(&p->sighand->siglock);
        return 0;
}

#ifdef CONFIG_DYNAMIC_SIGFRAME
static inline void sigaltstack_lock(void)
        __acquires(&current->sighand->siglock)
{
        spin_lock_irq(&current->sighand->siglock);
}

static inline void sigaltstack_unlock(void)
        __releases(&current->sighand->siglock)
{
        spin_unlock_irq(&current->sighand->siglock);
}
#else
static inline void sigaltstack_lock(void) { }
static inline void sigaltstack_unlock(void) { }
#endif

static int
do_sigaltstack (const stack_t *ss, stack_t *oss, unsigned long sp,
                size_t min_ss_size)
{
        struct task_struct *t = current;
        int ret = 0;

        if (oss) {
                memset(oss, 0, sizeof(stack_t));
                oss->ss_sp = (void __user *) t->sas_ss_sp;
                oss->ss_size = t->sas_ss_size;
                oss->ss_flags = sas_ss_flags(sp) |
                        (current->sas_ss_flags & SS_FLAG_BITS);
        }

        if (ss) {
                void __user *ss_sp = ss->ss_sp;
                size_t ss_size = ss->ss_size;
                unsigned ss_flags = ss->ss_flags;
                int ss_mode;

                if (unlikely(on_sig_stack(sp)))
                        return -EPERM;

                ss_mode = ss_flags & ~SS_FLAG_BITS;
                if (unlikely(ss_mode != SS_DISABLE && ss_mode != SS_ONSTACK &&
                                ss_mode != 0))
                        return -EINVAL;

                /*
                 * Return before taking any locks if no actual
                 * sigaltstack changes were requested.
                 */
                if (t->sas_ss_sp == (unsigned long)ss_sp &&
                    t->sas_ss_size == ss_size &&
                    t->sas_ss_flags == ss_flags)
                        return 0;

                sigaltstack_lock();
                if (ss_mode == SS_DISABLE) {
                        ss_size = 0;
                        ss_sp = NULL;
                } else {
                        if (unlikely(ss_size < min_ss_size))
                                ret = -ENOMEM;
                        if (!sigaltstack_size_valid(ss_size))
                                ret = -ENOMEM;
                }
                if (!ret) {
                        t->sas_ss_sp = (unsigned long) ss_sp;
                        t->sas_ss_size = ss_size;
                        t->sas_ss_flags = ss_flags;
                }
                sigaltstack_unlock();
        }
        return ret;
}

SYSCALL_DEFINE2(sigaltstack,const stack_t __user *,uss, stack_t __user *,uoss)
{
        stack_t new, old;
        int err;
        if (uss && copy_from_user(&new, uss, sizeof(stack_t)))
                return -EFAULT;
        err = do_sigaltstack(uss ? &new : NULL, uoss ? &old : NULL,
                              current_user_stack_pointer(),
                              MINSIGSTKSZ);
        if (!err && uoss && copy_to_user(uoss, &old, sizeof(stack_t)))
                err = -EFAULT;
        return err;
}

int restore_altstack(const stack_t __user *uss)
{
        stack_t new;
        if (copy_from_user(&new, uss, sizeof(stack_t)))
                return -EFAULT;
        (void)do_sigaltstack(&new, NULL, current_user_stack_pointer(),
                             MINSIGSTKSZ);
        /* squash all but EFAULT for now */
        return 0;
}

int __save_altstack(stack_t __user *uss, unsigned long sp)
{
        struct task_struct *t = current;
        int err = __put_user((void __user *)t->sas_ss_sp, &uss->ss_sp) |
                __put_user(t->sas_ss_flags, &uss->ss_flags) |
                __put_user(t->sas_ss_size, &uss->ss_size);
        return err;
}

#ifdef CONFIG_COMPAT
static int do_compat_sigaltstack(const compat_stack_t __user *uss_ptr,
                                 compat_stack_t __user *uoss_ptr)
{
        stack_t uss, uoss;
        int ret;

        if (uss_ptr) {
                compat_stack_t uss32;
                if (copy_from_user(&uss32, uss_ptr, sizeof(compat_stack_t)))
                        return -EFAULT;
                uss.ss_sp = compat_ptr(uss32.ss_sp);
                uss.ss_flags = uss32.ss_flags;
                uss.ss_size = uss32.ss_size;
        }
        ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss,
                             compat_user_stack_pointer(),
                             COMPAT_MINSIGSTKSZ);
        if (ret >= 0 && uoss_ptr)  {
                compat_stack_t old;
                memset(&old, 0, sizeof(old));
                old.ss_sp = ptr_to_compat(uoss.ss_sp);
                old.ss_flags = uoss.ss_flags;
                old.ss_size = uoss.ss_size;
                if (copy_to_user(uoss_ptr, &old, sizeof(compat_stack_t)))
                        ret = -EFAULT;
        }
        return ret;
}

COMPAT_SYSCALL_DEFINE2(sigaltstack,
                        const compat_stack_t __user *, uss_ptr,
                        compat_stack_t __user *, uoss_ptr)
{
        return do_compat_sigaltstack(uss_ptr, uoss_ptr);
}

int compat_restore_altstack(const compat_stack_t __user *uss)
{
        int err = do_compat_sigaltstack(uss, NULL);
        /* squash all but -EFAULT for now */
        return err == -EFAULT ? err : 0;
}

int __compat_save_altstack(compat_stack_t __user *uss, unsigned long sp)
{
        int err;
        struct task_struct *t = current;
        err = __put_user(ptr_to_compat((void __user *)t->sas_ss_sp),
                         &uss->ss_sp) |
                __put_user(t->sas_ss_flags, &uss->ss_flags) |
                __put_user(t->sas_ss_size, &uss->ss_size);
        return err;
}
#endif

#ifdef __ARCH_WANT_SYS_SIGPENDING

/**
 *  sys_sigpending - examine pending signals
 *  @uset: where mask of pending signal is returned
 */
SYSCALL_DEFINE1(sigpending, old_sigset_t __user *, uset)
{
        sigset_t set;

        if (sizeof(old_sigset_t) > sizeof(*uset))
                return -EINVAL;

        do_sigpending(&set);

        if (copy_to_user(uset, &set, sizeof(old_sigset_t)))
                return -EFAULT;

        return 0;
}

#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE1(sigpending, compat_old_sigset_t __user *, set32)
{
        sigset_t set;

        do_sigpending(&set);

        return put_user(set.sig[0], set32);
}
#endif

#endif

#ifdef __ARCH_WANT_SYS_SIGPROCMASK
/**
 *  sys_sigprocmask - examine and change blocked signals
 *  @how: whether to add, remove, or set signals
 *  @nset: signals to add or remove (if non-null)
 *  @oset: previous value of signal mask if non-null
 *
 * Some platforms have their own version with special arguments;
 * others support only sys_rt_sigprocmask.
 */

SYSCALL_DEFINE3(sigprocmask, int, how, old_sigset_t __user *, nset,
                old_sigset_t __user *, oset)
{
        old_sigset_t old_set, new_set;
        sigset_t new_blocked;

        old_set = current->blocked.sig[0];

        if (nset) {
                if (copy_from_user(&new_set, nset, sizeof(*nset)))
                        return -EFAULT;

                new_blocked = current->blocked;

                switch (how) {
                case SIG_BLOCK:
                        sigaddsetmask(&new_blocked, new_set);
                        break;
                case SIG_UNBLOCK:
                        sigdelsetmask(&new_blocked, new_set);
                        break;
                case SIG_SETMASK:
                        new_blocked.sig[0] = new_set;
                        break;
                default:
                        return -EINVAL;
                }

                set_current_blocked(&new_blocked);
        }

        if (oset) {
                if (copy_to_user(oset, &old_set, sizeof(*oset)))
                        return -EFAULT;
        }

        return 0;
}
#endif /* __ARCH_WANT_SYS_SIGPROCMASK */

#ifndef CONFIG_ODD_RT_SIGACTION
/**
 *  sys_rt_sigaction - alter an action taken by a process
 *  @sig: signal to be sent
 *  @act: new sigaction
 *  @oact: used to save the previous sigaction
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE4(rt_sigaction, int, sig,
                const struct sigaction __user *, act,
                struct sigaction __user *, oact,
                size_t, sigsetsize)
{
        struct k_sigaction new_sa, old_sa;
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (act && copy_from_user(&new_sa.sa, act, sizeof(new_sa.sa)))
                return -EFAULT;

        ret = do_sigaction(sig, act ? &new_sa : NULL, oact ? &old_sa : NULL);
        if (ret)
                return ret;

        if (oact && copy_to_user(oact, &old_sa.sa, sizeof(old_sa.sa)))
                return -EFAULT;

        return 0;
}
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(rt_sigaction, int, sig,
                const struct compat_sigaction __user *, act,
                struct compat_sigaction __user *, oact,
                compat_size_t, sigsetsize)
{
        struct k_sigaction new_ka, old_ka;
#ifdef __ARCH_HAS_SA_RESTORER
        compat_uptr_t restorer;
#endif
        int ret;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(compat_sigset_t))
                return -EINVAL;

        if (act) {
                compat_uptr_t handler;
                ret = get_user(handler, &act->sa_handler);
                new_ka.sa.sa_handler = compat_ptr(handler);
#ifdef __ARCH_HAS_SA_RESTORER
                ret |= get_user(restorer, &act->sa_restorer);
                new_ka.sa.sa_restorer = compat_ptr(restorer);
#endif
                ret |= get_compat_sigset(&new_ka.sa.sa_mask, &act->sa_mask);
                ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags);
                if (ret)
                        return -EFAULT;
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);
        if (!ret && oact) {
                ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), 
                               &oact->sa_handler);
                ret |= put_compat_sigset(&oact->sa_mask, &old_ka.sa.sa_mask,
                                         sizeof(oact->sa_mask));
                ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags);
#ifdef __ARCH_HAS_SA_RESTORER
                ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer),
                                &oact->sa_restorer);
#endif
        }
        return ret;
}
#endif
#endif /* !CONFIG_ODD_RT_SIGACTION */

#ifdef CONFIG_OLD_SIGACTION
SYSCALL_DEFINE3(sigaction, int, sig,
                const struct old_sigaction __user *, act,
                struct old_sigaction __user *, oact)
{
        struct k_sigaction new_ka, old_ka;
        int ret;

        if (act) {
                old_sigset_t mask;
                if (!access_ok(act, sizeof(*act)) ||
                    __get_user(new_ka.sa.sa_handler, &act->sa_handler) ||
                    __get_user(new_ka.sa.sa_restorer, &act->sa_restorer) ||
                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
                    __get_user(mask, &act->sa_mask))
                        return -EFAULT;
#ifdef __ARCH_HAS_KA_RESTORER
                new_ka.ka_restorer = NULL;
#endif
                siginitset(&new_ka.sa.sa_mask, mask);
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);

        if (!ret && oact) {
                if (!access_ok(oact, sizeof(*oact)) ||
                    __put_user(old_ka.sa.sa_handler, &oact->sa_handler) ||
                    __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer) ||
                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
                        return -EFAULT;
        }

        return ret;
}
#endif
#ifdef CONFIG_COMPAT_OLD_SIGACTION
COMPAT_SYSCALL_DEFINE3(sigaction, int, sig,
                const struct compat_old_sigaction __user *, act,
                struct compat_old_sigaction __user *, oact)
{
        struct k_sigaction new_ka, old_ka;
        int ret;
        compat_old_sigset_t mask;
        compat_uptr_t handler, restorer;

        if (act) {
                if (!access_ok(act, sizeof(*act)) ||
                    __get_user(handler, &act->sa_handler) ||
                    __get_user(restorer, &act->sa_restorer) ||
                    __get_user(new_ka.sa.sa_flags, &act->sa_flags) ||
                    __get_user(mask, &act->sa_mask))
                        return -EFAULT;

#ifdef __ARCH_HAS_KA_RESTORER
                new_ka.ka_restorer = NULL;
#endif
                new_ka.sa.sa_handler = compat_ptr(handler);
                new_ka.sa.sa_restorer = compat_ptr(restorer);
                siginitset(&new_ka.sa.sa_mask, mask);
        }

        ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL);

        if (!ret && oact) {
                if (!access_ok(oact, sizeof(*oact)) ||
                    __put_user(ptr_to_compat(old_ka.sa.sa_handler),
                               &oact->sa_handler) ||
                    __put_user(ptr_to_compat(old_ka.sa.sa_restorer),
                               &oact->sa_restorer) ||
                    __put_user(old_ka.sa.sa_flags, &oact->sa_flags) ||
                    __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask))
                        return -EFAULT;
        }
        return ret;
}
#endif

#ifdef CONFIG_SGETMASK_SYSCALL

/*
 * For backwards compatibility.  Functionality superseded by sigprocmask.
 */
SYSCALL_DEFINE0(sgetmask)
{
        /* SMP safe */
        return current->blocked.sig[0];
}

SYSCALL_DEFINE1(ssetmask, int, newmask)
{
        int old = current->blocked.sig[0];
        sigset_t newset;

        siginitset(&newset, newmask);
        set_current_blocked(&newset);

        return old;
}
#endif /* CONFIG_SGETMASK_SYSCALL */

#ifdef __ARCH_WANT_SYS_SIGNAL
/*
 * For backwards compatibility.  Functionality superseded by sigaction.
 */
SYSCALL_DEFINE2(signal, int, sig, __sighandler_t, handler)
{
        struct k_sigaction new_sa, old_sa;
        int ret;

        new_sa.sa.sa_handler = handler;
        new_sa.sa.sa_flags = SA_ONESHOT | SA_NOMASK;
        sigemptyset(&new_sa.sa.sa_mask);

        ret = do_sigaction(sig, &new_sa, &old_sa);

        return ret ? ret : (unsigned long)old_sa.sa.sa_handler;
}
#endif /* __ARCH_WANT_SYS_SIGNAL */

#ifdef __ARCH_WANT_SYS_PAUSE

SYSCALL_DEFINE0(pause)
{
        while (!signal_pending(current)) {
                __set_current_state(TASK_INTERRUPTIBLE);
                schedule();
        }
        return -ERESTARTNOHAND;
}

#endif

static int sigsuspend(sigset_t *set)
{
        current->saved_sigmask = current->blocked;
        set_current_blocked(set);

        while (!signal_pending(current)) {
                __set_current_state(TASK_INTERRUPTIBLE);
                schedule();
        }
        set_restore_sigmask();
        return -ERESTARTNOHAND;
}

/**
 *  sys_rt_sigsuspend - replace the signal mask for a value with the
 *        @unewset value until a signal is received
 *  @unewset: new signal mask value
 *  @sigsetsize: size of sigset_t type
 */
SYSCALL_DEFINE2(rt_sigsuspend, sigset_t __user *, unewset, size_t, sigsetsize)
{
        sigset_t newset;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (copy_from_user(&newset, unewset, sizeof(newset)))
                return -EFAULT;
        return sigsuspend(&newset);
}
 
#ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE2(rt_sigsuspend, compat_sigset_t __user *, unewset, compat_size_t, sigsetsize)
{
        sigset_t newset;

        /* XXX: Don't preclude handling different sized sigset_t's.  */
        if (sigsetsize != sizeof(sigset_t))
                return -EINVAL;

        if (get_compat_sigset(&newset, unewset))
                return -EFAULT;
        return sigsuspend(&newset);
}
#endif

#ifdef CONFIG_OLD_SIGSUSPEND
SYSCALL_DEFINE1(sigsuspend, old_sigset_t, mask)
{
        sigset_t blocked;
        siginitset(&blocked, mask);
        return sigsuspend(&blocked);
}
#endif
#ifdef CONFIG_OLD_SIGSUSPEND3
SYSCALL_DEFINE3(sigsuspend, int, unused1, int, unused2, old_sigset_t, mask)
{
        sigset_t blocked;
        siginitset(&blocked, mask);
        return sigsuspend(&blocked);
}
#endif

__weak const char *arch_vma_name(struct vm_area_struct *vma)
{
        return NULL;
}

static inline void siginfo_buildtime_checks(void)
{
        BUILD_BUG_ON(sizeof(struct siginfo) != SI_MAX_SIZE);

        /* Verify the offsets in the two siginfos match */
#define CHECK_OFFSET(field) \
        BUILD_BUG_ON(offsetof(siginfo_t, field) != offsetof(kernel_siginfo_t, field))

        /* kill */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);

        /* timer */
        CHECK_OFFSET(si_tid);
        CHECK_OFFSET(si_overrun);
        CHECK_OFFSET(si_value);

        /* rt */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);
        CHECK_OFFSET(si_value);

        /* sigchld */
        CHECK_OFFSET(si_pid);
        CHECK_OFFSET(si_uid);
        CHECK_OFFSET(si_status);
        CHECK_OFFSET(si_utime);
        CHECK_OFFSET(si_stime);

        /* sigfault */
        CHECK_OFFSET(si_addr);
        CHECK_OFFSET(si_trapno);
        CHECK_OFFSET(si_addr_lsb);
        CHECK_OFFSET(si_lower);
        CHECK_OFFSET(si_upper);
        CHECK_OFFSET(si_pkey);
        CHECK_OFFSET(si_perf_data);
        CHECK_OFFSET(si_perf_type);
        CHECK_OFFSET(si_perf_flags);

        /* sigpoll */
        CHECK_OFFSET(si_band);
        CHECK_OFFSET(si_fd);

        /* sigsys */
        CHECK_OFFSET(si_call_addr);
        CHECK_OFFSET(si_syscall);
        CHECK_OFFSET(si_arch);
#undef CHECK_OFFSET

        /* usb asyncio */
        BUILD_BUG_ON(offsetof(struct siginfo, si_pid) !=
                     offsetof(struct siginfo, si_addr));
        if (sizeof(int) == sizeof(void __user *)) {
                BUILD_BUG_ON(sizeof_field(struct siginfo, si_pid) !=
                             sizeof(void __user *));
        } else {
                BUILD_BUG_ON((sizeof_field(struct siginfo, si_pid) +
                              sizeof_field(struct siginfo, si_uid)) !=
                             sizeof(void __user *));
                BUILD_BUG_ON(offsetofend(struct siginfo, si_pid) !=
                             offsetof(struct siginfo, si_uid));
        }
#ifdef CONFIG_COMPAT
        BUILD_BUG_ON(offsetof(struct compat_siginfo, si_pid) !=
                     offsetof(struct compat_siginfo, si_addr));
        BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
                     sizeof(compat_uptr_t));
        BUILD_BUG_ON(sizeof_field(struct compat_siginfo, si_pid) !=
                     sizeof_field(struct siginfo, si_pid));
#endif
}

#if defined(CONFIG_SYSCTL)
static const struct ctl_table signal_debug_table[] = {
#ifdef CONFIG_SYSCTL_EXCEPTION_TRACE
        {
                .procname        = "exception-trace",
                .data                = &show_unhandled_signals,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec
        },
#endif
};

static const struct ctl_table signal_table[] = {
        {
                .procname        = "print-fatal-signals",
                .data                = &print_fatal_signals,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec,
        },
};

static int __init init_signal_sysctls(void)
{
        register_sysctl_init("debug", signal_debug_table);
        register_sysctl_init("kernel", signal_table);
        return 0;
}
early_initcall(init_signal_sysctls);
#endif /* CONFIG_SYSCTL */

void __init signals_init(void)
{
        siginfo_buildtime_checks();

        sigqueue_cachep = KMEM_CACHE(sigqueue, SLAB_PANIC | SLAB_ACCOUNT);
}

#ifdef CONFIG_KGDB_KDB
#include <linux/kdb.h>
/*
 * kdb_send_sig - Allows kdb to send signals without exposing
 * signal internals.  This function checks if the required locks are
 * available before calling the main signal code, to avoid kdb
 * deadlocks.
 */
void kdb_send_sig(struct task_struct *t, int sig)
{
        static struct task_struct *kdb_prev_t;
        int new_t, ret;
        if (!spin_trylock(&t->sighand->siglock)) {
                kdb_printf("Can't do kill command now.\n"
                           "The sigmask lock is held somewhere else in "
                           "kernel, try again later\n");
                return;
        }
        new_t = kdb_prev_t != t;
        kdb_prev_t = t;
        if (!task_is_running(t) && new_t) {
                spin_unlock(&t->sighand->siglock);
                kdb_printf("Process is not RUNNING, sending a signal from "
                           "kdb risks deadlock\n"
                           "on the run queue locks. "
                           "The signal has _not_ been sent.\n"
                           "Reissue the kill command if you want to risk "
                           "the deadlock.\n");
                return;
        }
        ret = send_signal_locked(sig, SEND_SIG_PRIV, t, PIDTYPE_PID);
        spin_unlock(&t->sighand->siglock);
        if (ret)
                kdb_printf("Fail to deliver Signal %d to process %d.\n",
                           sig, t->pid);
        else
                kdb_printf("Signal %d is sent to process %d.\n", sig, t->pid);
}
#endif        /* CONFIG_KGDB_KDB */

















































































































   10 




    2 





    3 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_MROUTE6_H
#define __LINUX_MROUTE6_H


#include <linux/pim.h>
#include <linux/skbuff.h>        /* for struct sk_buff_head */
#include <net/net_namespace.h>
#include <uapi/linux/mroute6.h>
#include <linux/mroute_base.h>
#include <linux/sockptr.h>
#include <net/fib_rules.h>

#ifdef CONFIG_IPV6_MROUTE
static inline int ip6_mroute_opt(int opt)
{
        return (opt >= MRT6_BASE) && (opt <= MRT6_MAX);
}
#else
static inline int ip6_mroute_opt(int opt)
{
        return 0;
}
#endif

struct sock;

#ifdef CONFIG_IPV6_MROUTE
extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);
extern int ip6_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t);
extern int ip6_mr_input(struct sk_buff *skb);
extern int ip6mr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg);
extern int ip6_mr_init(void);
extern int ip6_mr_output(struct net *net, struct sock *sk, struct sk_buff *skb);
extern void ip6_mr_cleanup(void);
int ip6mr_ioctl(struct sock *sk, int cmd, void *arg);
#else
static inline int ip6_mroute_setsockopt(struct sock *sock, int optname,
                sockptr_t optval, unsigned int optlen)
{
        return -ENOPROTOOPT;
}

static inline
int ip6_mroute_getsockopt(struct sock *sock,
                          int optname, sockptr_t optval, sockptr_t optlen)
{
        return -ENOPROTOOPT;
}

static inline
int ip6mr_ioctl(struct sock *sk, int cmd, void *arg)
{
        return -ENOIOCTLCMD;
}

static inline int ip6_mr_init(void)
{
        return 0;
}

static inline int
ip6_mr_output(struct net *net, struct sock *sk, struct sk_buff *skb)
{
        return ip6_output(net, sk, skb);
}

static inline void ip6_mr_cleanup(void)
{
        return;
}
#endif

#ifdef CONFIG_IPV6_MROUTE_MULTIPLE_TABLES
bool ip6mr_rule_default(const struct fib_rule *rule);
#else
static inline bool ip6mr_rule_default(const struct fib_rule *rule)
{
        return true;
}
#endif

#define VIFF_STATIC 0x8000

struct mfc6_cache_cmp_arg {
        struct in6_addr mf6c_mcastgrp;
        struct in6_addr mf6c_origin;
};

struct mfc6_cache {
        struct mr_mfc _c;
        union {
                struct {
                        struct in6_addr mf6c_mcastgrp;
                        struct in6_addr mf6c_origin;
                };
                struct mfc6_cache_cmp_arg cmparg;
        };
};

#define MFC_ASSERT_THRESH (3*HZ)                /* Maximal freq. of asserts */

struct rtmsg;
extern int ip6mr_get_route(struct net *net, struct sk_buff *skb,
                           struct rtmsg *rtm, u32 portid);

#ifdef CONFIG_IPV6_MROUTE
bool mroute6_is_socket(struct net *net, struct sk_buff *skb);
extern int ip6mr_sk_done(struct sock *sk);
static inline int ip6mr_sk_ioctl(struct sock *sk, unsigned int cmd,
                                 void __user *arg)
{
        switch (cmd) {
        /* These userspace buffers will be consumed by ip6mr_ioctl() */
        case SIOCGETMIFCNT_IN6: {
                struct sioc_mif_req6 buffer;

                return sock_ioctl_inout(sk, cmd, arg, &buffer,
                                        sizeof(buffer));
                }
        case SIOCGETSGCNT_IN6: {
                struct sioc_sg_req6 buffer;

                return sock_ioctl_inout(sk, cmd, arg, &buffer,
                                        sizeof(buffer));
                }
        }

        return 1;
}
#else
static inline bool mroute6_is_socket(struct net *net, struct sk_buff *skb)
{
        return false;
}
static inline int ip6mr_sk_done(struct sock *sk)
{
        return 0;
}

static inline int ip6mr_sk_ioctl(struct sock *sk, unsigned int cmd,
                                 void __user *arg)
{
        return 1;
}
#endif
#endif































































































































































































































    1 










    2 




    3 




    1 

    2 


    2 



























    3 


    3 











    1 









   13 



















    1 
    1 

    1 












    1 





   13 
   13 



    1 
    1 

    1 











   13 

















    1 













    1 










    1 





    1 











    1 







    1 

    1 




































    1 










    2 


    2 
    3 









    2 






















    1 












    3 
    3 
    3 








    3 




    3 








    3 
    3 
    3 









    1 



    2 










   13 








































































































































































































    4 











    1 


    3 

    3 
    2 

    1 



    1 



























   42 




   13 
   13 



   13 



   13 



   42 



   42 

   42 
   42 



















   42 
















   13 
   13 
   13 


   13 

   13 











   13 
   13 



   13 


   13 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
// SPDX-License-Identifier: GPL-2.0-only
/*
 *        IEEE 802.1Q Multiple Registration Protocol (MRP)
 *
 *        Copyright (c) 2012 Massachusetts Institute of Technology
 *
 *        Adapted from code in net/802/garp.c
 *        Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 */
#include <linux/kernel.h>
#include <linux/timer.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <net/mrp.h>
#include <linux/unaligned.h>

static unsigned int mrp_join_time __read_mostly = 200;
module_param(mrp_join_time, uint, 0644);
MODULE_PARM_DESC(mrp_join_time, "Join time in ms (default 200ms)");

static unsigned int mrp_periodic_time __read_mostly = 1000;
module_param(mrp_periodic_time, uint, 0644);
MODULE_PARM_DESC(mrp_periodic_time, "Periodic time in ms (default 1s)");

MODULE_DESCRIPTION("IEEE 802.1Q Multiple Registration Protocol (MRP)");
MODULE_LICENSE("GPL");

static const u8
mrp_applicant_state_table[MRP_APPLICANT_MAX + 1][MRP_EVENT_MAX + 1] = {
        [MRP_APPLICANT_VO] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_VP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_VO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_VO,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VO,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VO,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_VO,
        },
        [MRP_APPLICANT_VP] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_VP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_VO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_AA,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VP,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VP,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_VP,
        },
        [MRP_APPLICANT_VN] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_VN,
                [MRP_EVENT_LV]                = MRP_APPLICANT_LA,
                [MRP_EVENT_TX]                = MRP_APPLICANT_AN,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VN,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VN,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_VN,
        },
        [MRP_APPLICANT_AN] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_AN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_AN,
                [MRP_EVENT_LV]                = MRP_APPLICANT_LA,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QA,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_AN,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_AN,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_AN,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AN,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AN,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VN,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VN,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VN,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AN,
        },
        [MRP_APPLICANT_AA] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_AA,
                [MRP_EVENT_LV]                = MRP_APPLICANT_LA,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QA,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QA,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VP,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VP,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AA,
        },
        [MRP_APPLICANT_QA] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_QA,
                [MRP_EVENT_LV]                = MRP_APPLICANT_LA,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QA,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_QA,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QA,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_QA,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AA,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VP,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VP,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AA,
        },
        [MRP_APPLICANT_LA] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_AA,
                [MRP_EVENT_LV]                = MRP_APPLICANT_LA,
                [MRP_EVENT_TX]                = MRP_APPLICANT_VO,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_LA,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_LA,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_LA,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_LA,
        },
        [MRP_APPLICANT_AO] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_AP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_AO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_AO,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QO,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VO,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VO,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AO,
        },
        [MRP_APPLICANT_QO] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_QP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_QO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QO,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_QO,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QO,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_QO,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AO,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VO,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VO,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VO,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_QO,
        },
        [MRP_APPLICANT_AP] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_AP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_AO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QA,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QP,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VP,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VP,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AP,
        },
        [MRP_APPLICANT_QP] = {
                [MRP_EVENT_NEW]                = MRP_APPLICANT_VN,
                [MRP_EVENT_JOIN]        = MRP_APPLICANT_QP,
                [MRP_EVENT_LV]                = MRP_APPLICANT_QO,
                [MRP_EVENT_TX]                = MRP_APPLICANT_QP,
                [MRP_EVENT_R_NEW]        = MRP_APPLICANT_QP,
                [MRP_EVENT_R_JOIN_IN]        = MRP_APPLICANT_QP,
                [MRP_EVENT_R_IN]        = MRP_APPLICANT_QP,
                [MRP_EVENT_R_JOIN_MT]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_MT]        = MRP_APPLICANT_AP,
                [MRP_EVENT_R_LV]        = MRP_APPLICANT_VP,
                [MRP_EVENT_R_LA]        = MRP_APPLICANT_VP,
                [MRP_EVENT_REDECLARE]        = MRP_APPLICANT_VP,
                [MRP_EVENT_PERIODIC]        = MRP_APPLICANT_AP,
        },
};

static const u8
mrp_tx_action_table[MRP_APPLICANT_MAX + 1] = {
        [MRP_APPLICANT_VO] = MRP_TX_ACTION_S_IN_OPTIONAL,
        [MRP_APPLICANT_VP] = MRP_TX_ACTION_S_JOIN_IN,
        [MRP_APPLICANT_VN] = MRP_TX_ACTION_S_NEW,
        [MRP_APPLICANT_AN] = MRP_TX_ACTION_S_NEW,
        [MRP_APPLICANT_AA] = MRP_TX_ACTION_S_JOIN_IN,
        [MRP_APPLICANT_QA] = MRP_TX_ACTION_S_JOIN_IN_OPTIONAL,
        [MRP_APPLICANT_LA] = MRP_TX_ACTION_S_LV,
        [MRP_APPLICANT_AO] = MRP_TX_ACTION_S_IN_OPTIONAL,
        [MRP_APPLICANT_QO] = MRP_TX_ACTION_S_IN_OPTIONAL,
        [MRP_APPLICANT_AP] = MRP_TX_ACTION_S_JOIN_IN,
        [MRP_APPLICANT_QP] = MRP_TX_ACTION_S_IN_OPTIONAL,
};

static void mrp_attrvalue_inc(void *value, u8 len)
{
        u8 *v = (u8 *)value;

        /* Add 1 to the last byte. If it becomes zero,
         * go to the previous byte and repeat.
         */
        while (len > 0 && !++v[--len])
                ;
}

static int mrp_attr_cmp(const struct mrp_attr *attr,
                         const void *value, u8 len, u8 type)
{
        if (attr->type != type)
                return attr->type - type;
        if (attr->len != len)
                return attr->len - len;
        return memcmp(attr->value, value, len);
}

static struct mrp_attr *mrp_attr_lookup(const struct mrp_applicant *app,
                                        const void *value, u8 len, u8 type)
{
        struct rb_node *parent = app->mad.rb_node;
        struct mrp_attr *attr;
        int d;

        while (parent) {
                attr = rb_entry(parent, struct mrp_attr, node);
                d = mrp_attr_cmp(attr, value, len, type);
                if (d > 0)
                        parent = parent->rb_left;
                else if (d < 0)
                        parent = parent->rb_right;
                else
                        return attr;
        }
        return NULL;
}

static struct mrp_attr *mrp_attr_create(struct mrp_applicant *app,
                                        const void *value, u8 len, u8 type)
{
        struct rb_node *parent = NULL, **p = &app->mad.rb_node;
        struct mrp_attr *attr;
        int d;

        while (*p) {
                parent = *p;
                attr = rb_entry(parent, struct mrp_attr, node);
                d = mrp_attr_cmp(attr, value, len, type);
                if (d > 0)
                        p = &parent->rb_left;
                else if (d < 0)
                        p = &parent->rb_right;
                else {
                        /* The attribute already exists; re-use it. */
                        return attr;
                }
        }
        attr = kmalloc(sizeof(*attr) + len, GFP_ATOMIC);
        if (!attr)
                return attr;
        attr->state = MRP_APPLICANT_VO;
        attr->type  = type;
        attr->len   = len;
        memcpy(attr->value, value, len);

        rb_link_node(&attr->node, parent, p);
        rb_insert_color(&attr->node, &app->mad);
        return attr;
}

static void mrp_attr_destroy(struct mrp_applicant *app, struct mrp_attr *attr)
{
        rb_erase(&attr->node, &app->mad);
        kfree(attr);
}

static void mrp_attr_destroy_all(struct mrp_applicant *app)
{
        struct rb_node *node, *next;
        struct mrp_attr *attr;

        for (node = rb_first(&app->mad);
             next = node ? rb_next(node) : NULL, node != NULL;
             node = next) {
                attr = rb_entry(node, struct mrp_attr, node);
                mrp_attr_destroy(app, attr);
        }
}

static int mrp_pdu_init(struct mrp_applicant *app)
{
        struct sk_buff *skb;
        struct mrp_pdu_hdr *ph;

        skb = alloc_skb(app->dev->mtu + LL_RESERVED_SPACE(app->dev),
                        GFP_ATOMIC);
        if (!skb)
                return -ENOMEM;

        skb->dev = app->dev;
        skb->protocol = app->app->pkttype.type;
        skb_reserve(skb, LL_RESERVED_SPACE(app->dev));
        skb_reset_network_header(skb);
        skb_reset_transport_header(skb);

        ph = __skb_put(skb, sizeof(*ph));
        ph->version = app->app->version;

        app->pdu = skb;
        return 0;
}

static int mrp_pdu_append_end_mark(struct mrp_applicant *app)
{
        __be16 *endmark;

        if (skb_tailroom(app->pdu) < sizeof(*endmark))
                return -1;
        endmark = __skb_put(app->pdu, sizeof(*endmark));
        put_unaligned(MRP_END_MARK, endmark);
        return 0;
}

static void mrp_pdu_queue(struct mrp_applicant *app)
{
        if (!app->pdu)
                return;

        if (mrp_cb(app->pdu)->mh)
                mrp_pdu_append_end_mark(app);
        mrp_pdu_append_end_mark(app);

        dev_hard_header(app->pdu, app->dev, ntohs(app->app->pkttype.type),
                        app->app->group_address, app->dev->dev_addr,
                        app->pdu->len);

        skb_queue_tail(&app->queue, app->pdu);
        app->pdu = NULL;
}

static void mrp_queue_xmit(struct mrp_applicant *app)
{
        struct sk_buff *skb;

        while ((skb = skb_dequeue(&app->queue)))
                dev_queue_xmit(skb);
}

static int mrp_pdu_append_msg_hdr(struct mrp_applicant *app,
                                  u8 attrtype, u8 attrlen)
{
        struct mrp_msg_hdr *mh;

        if (mrp_cb(app->pdu)->mh) {
                if (mrp_pdu_append_end_mark(app) < 0)
                        return -1;
                mrp_cb(app->pdu)->mh = NULL;
                mrp_cb(app->pdu)->vah = NULL;
        }

        if (skb_tailroom(app->pdu) < sizeof(*mh))
                return -1;
        mh = __skb_put(app->pdu, sizeof(*mh));
        mh->attrtype = attrtype;
        mh->attrlen = attrlen;
        mrp_cb(app->pdu)->mh = mh;
        return 0;
}

static int mrp_pdu_append_vecattr_hdr(struct mrp_applicant *app,
                                      const void *firstattrvalue, u8 attrlen)
{
        struct mrp_vecattr_hdr *vah;

        if (skb_tailroom(app->pdu) < sizeof(*vah) + attrlen)
                return -1;
        vah = __skb_put(app->pdu, sizeof(*vah) + attrlen);
        put_unaligned(0, &vah->lenflags);
        memcpy(vah->firstattrvalue, firstattrvalue, attrlen);
        mrp_cb(app->pdu)->vah = vah;
        memcpy(mrp_cb(app->pdu)->attrvalue, firstattrvalue, attrlen);
        return 0;
}

static int mrp_pdu_append_vecattr_event(struct mrp_applicant *app,
                                        const struct mrp_attr *attr,
                                        enum mrp_vecattr_event vaevent)
{
        u16 len, pos;
        u8 *vaevents;
        int err;
again:
        if (!app->pdu) {
                err = mrp_pdu_init(app);
                if (err < 0)
                        return err;
        }

        /* If there is no Message header in the PDU, or the Message header is
         * for a different attribute type, add an EndMark (if necessary) and a
         * new Message header to the PDU.
         */
        if (!mrp_cb(app->pdu)->mh ||
            mrp_cb(app->pdu)->mh->attrtype != attr->type ||
            mrp_cb(app->pdu)->mh->attrlen != attr->len) {
                if (mrp_pdu_append_msg_hdr(app, attr->type, attr->len) < 0)
                        goto queue;
        }

        /* If there is no VectorAttribute header for this Message in the PDU,
         * or this attribute's value does not sequentially follow the previous
         * attribute's value, add a new VectorAttribute header to the PDU.
         */
        if (!mrp_cb(app->pdu)->vah ||
            memcmp(mrp_cb(app->pdu)->attrvalue, attr->value, attr->len)) {
                if (mrp_pdu_append_vecattr_hdr(app, attr->value, attr->len) < 0)
                        goto queue;
        }

        len = be16_to_cpu(get_unaligned(&mrp_cb(app->pdu)->vah->lenflags));
        pos = len % 3;

        /* Events are packed into Vectors in the PDU, three to a byte. Add a
         * byte to the end of the Vector if necessary.
         */
        if (!pos) {
                if (skb_tailroom(app->pdu) < sizeof(u8))
                        goto queue;
                vaevents = __skb_put(app->pdu, sizeof(u8));
        } else {
                vaevents = (u8 *)(skb_tail_pointer(app->pdu) - sizeof(u8));
        }

        switch (pos) {
        case 0:
                *vaevents = vaevent * (__MRP_VECATTR_EVENT_MAX *
                                       __MRP_VECATTR_EVENT_MAX);
                break;
        case 1:
                *vaevents += vaevent * __MRP_VECATTR_EVENT_MAX;
                break;
        case 2:
                *vaevents += vaevent;
                break;
        default:
                WARN_ON(1);
        }

        /* Increment the length of the VectorAttribute in the PDU, as well as
         * the value of the next attribute that would continue its Vector.
         */
        put_unaligned(cpu_to_be16(++len), &mrp_cb(app->pdu)->vah->lenflags);
        mrp_attrvalue_inc(mrp_cb(app->pdu)->attrvalue, attr->len);

        return 0;

queue:
        mrp_pdu_queue(app);
        goto again;
}

static void mrp_attr_event(struct mrp_applicant *app,
                           struct mrp_attr *attr, enum mrp_event event)
{
        enum mrp_applicant_state state;

        state = mrp_applicant_state_table[attr->state][event];
        if (state == MRP_APPLICANT_INVALID) {
                WARN_ON(1);
                return;
        }

        if (event == MRP_EVENT_TX) {
                /* When appending the attribute fails, don't update its state
                 * in order to retry at the next TX event.
                 */

                switch (mrp_tx_action_table[attr->state]) {
                case MRP_TX_ACTION_NONE:
                case MRP_TX_ACTION_S_JOIN_IN_OPTIONAL:
                case MRP_TX_ACTION_S_IN_OPTIONAL:
                        break;
                case MRP_TX_ACTION_S_NEW:
                        if (mrp_pdu_append_vecattr_event(
                                    app, attr, MRP_VECATTR_EVENT_NEW) < 0)
                                return;
                        break;
                case MRP_TX_ACTION_S_JOIN_IN:
                        if (mrp_pdu_append_vecattr_event(
                                    app, attr, MRP_VECATTR_EVENT_JOIN_IN) < 0)
                                return;
                        break;
                case MRP_TX_ACTION_S_LV:
                        if (mrp_pdu_append_vecattr_event(
                                    app, attr, MRP_VECATTR_EVENT_LV) < 0)
                                return;
                        /* As a pure applicant, sending a leave message
                         * implies that the attribute was unregistered and
                         * can be destroyed.
                         */
                        mrp_attr_destroy(app, attr);
                        return;
                default:
                        WARN_ON(1);
                }
        }

        attr->state = state;
}

int mrp_request_join(const struct net_device *dev,
                     const struct mrp_application *appl,
                     const void *value, u8 len, u8 type)
{
        struct mrp_port *port = rtnl_dereference(dev->mrp_port);
        struct mrp_applicant *app = rtnl_dereference(
                port->applicants[appl->type]);
        struct mrp_attr *attr;

        if (sizeof(struct mrp_skb_cb) + len >
            sizeof_field(struct sk_buff, cb))
                return -ENOMEM;

        spin_lock_bh(&app->lock);
        attr = mrp_attr_create(app, value, len, type);
        if (!attr) {
                spin_unlock_bh(&app->lock);
                return -ENOMEM;
        }
        mrp_attr_event(app, attr, MRP_EVENT_JOIN);
        spin_unlock_bh(&app->lock);
        return 0;
}
EXPORT_SYMBOL_GPL(mrp_request_join);

void mrp_request_leave(const struct net_device *dev,
                       const struct mrp_application *appl,
                       const void *value, u8 len, u8 type)
{
        struct mrp_port *port = rtnl_dereference(dev->mrp_port);
        struct mrp_applicant *app = rtnl_dereference(
                port->applicants[appl->type]);
        struct mrp_attr *attr;

        if (sizeof(struct mrp_skb_cb) + len >
            sizeof_field(struct sk_buff, cb))
                return;

        spin_lock_bh(&app->lock);
        attr = mrp_attr_lookup(app, value, len, type);
        if (!attr) {
                spin_unlock_bh(&app->lock);
                return;
        }
        mrp_attr_event(app, attr, MRP_EVENT_LV);
        spin_unlock_bh(&app->lock);
}
EXPORT_SYMBOL_GPL(mrp_request_leave);

static void mrp_mad_event(struct mrp_applicant *app, enum mrp_event event)
{
        struct rb_node *node, *next;
        struct mrp_attr *attr;

        for (node = rb_first(&app->mad);
             next = node ? rb_next(node) : NULL, node != NULL;
             node = next) {
                attr = rb_entry(node, struct mrp_attr, node);
                mrp_attr_event(app, attr, event);
        }
}

static void mrp_join_timer_arm(struct mrp_applicant *app)
{
        unsigned long delay;

        delay = get_random_u32_below(msecs_to_jiffies(mrp_join_time));
        mod_timer(&app->join_timer, jiffies + delay);
}

static void mrp_join_timer(struct timer_list *t)
{
        struct mrp_applicant *app = timer_container_of(app, t, join_timer);

        spin_lock(&app->lock);
        mrp_mad_event(app, MRP_EVENT_TX);
        mrp_pdu_queue(app);
        spin_unlock(&app->lock);

        mrp_queue_xmit(app);
        spin_lock(&app->lock);
        if (likely(app->active))
                mrp_join_timer_arm(app);
        spin_unlock(&app->lock);
}

static void mrp_periodic_timer_arm(struct mrp_applicant *app)
{
        mod_timer(&app->periodic_timer,
                  jiffies + msecs_to_jiffies(mrp_periodic_time));
}

static void mrp_periodic_timer(struct timer_list *t)
{
        struct mrp_applicant *app = timer_container_of(app, t, periodic_timer);

        spin_lock(&app->lock);
        if (likely(app->active)) {
                mrp_mad_event(app, MRP_EVENT_PERIODIC);
                mrp_pdu_queue(app);
                mrp_periodic_timer_arm(app);
        }
        spin_unlock(&app->lock);
}

static int mrp_pdu_parse_end_mark(struct sk_buff *skb, int *offset)
{
        __be16 endmark;

        if (skb_copy_bits(skb, *offset, &endmark, sizeof(endmark)) < 0)
                return -1;
        if (endmark == MRP_END_MARK) {
                *offset += sizeof(endmark);
                return -1;
        }
        return 0;
}

static void mrp_pdu_parse_vecattr_event(struct mrp_applicant *app,
                                        struct sk_buff *skb,
                                        enum mrp_vecattr_event vaevent)
{
        struct mrp_attr *attr;
        enum mrp_event event;

        attr = mrp_attr_lookup(app, mrp_cb(skb)->attrvalue,
                               mrp_cb(skb)->mh->attrlen,
                               mrp_cb(skb)->mh->attrtype);
        if (attr == NULL)
                return;

        switch (vaevent) {
        case MRP_VECATTR_EVENT_NEW:
                event = MRP_EVENT_R_NEW;
                break;
        case MRP_VECATTR_EVENT_JOIN_IN:
                event = MRP_EVENT_R_JOIN_IN;
                break;
        case MRP_VECATTR_EVENT_IN:
                event = MRP_EVENT_R_IN;
                break;
        case MRP_VECATTR_EVENT_JOIN_MT:
                event = MRP_EVENT_R_JOIN_MT;
                break;
        case MRP_VECATTR_EVENT_MT:
                event = MRP_EVENT_R_MT;
                break;
        case MRP_VECATTR_EVENT_LV:
                event = MRP_EVENT_R_LV;
                break;
        default:
                return;
        }

        mrp_attr_event(app, attr, event);
}

static int mrp_pdu_parse_vecattr(struct mrp_applicant *app,
                                 struct sk_buff *skb, int *offset)
{
        struct mrp_vecattr_hdr _vah;
        u16 valen;
        u8 vaevents, vaevent;

        mrp_cb(skb)->vah = skb_header_pointer(skb, *offset, sizeof(_vah),
                                              &_vah);
        if (!mrp_cb(skb)->vah)
                return -1;
        *offset += sizeof(_vah);

        if (get_unaligned(&mrp_cb(skb)->vah->lenflags) &
            MRP_VECATTR_HDR_FLAG_LA)
                mrp_mad_event(app, MRP_EVENT_R_LA);
        valen = be16_to_cpu(get_unaligned(&mrp_cb(skb)->vah->lenflags) &
                            MRP_VECATTR_HDR_LEN_MASK);

        /* The VectorAttribute structure in a PDU carries event information
         * about one or more attributes having consecutive values. Only the
         * value for the first attribute is contained in the structure. So
         * we make a copy of that value, and then increment it each time we
         * advance to the next event in its Vector.
         */
        if (sizeof(struct mrp_skb_cb) + mrp_cb(skb)->mh->attrlen >
            sizeof_field(struct sk_buff, cb))
                return -1;
        if (skb_copy_bits(skb, *offset, mrp_cb(skb)->attrvalue,
                          mrp_cb(skb)->mh->attrlen) < 0)
                return -1;
        *offset += mrp_cb(skb)->mh->attrlen;

        /* In a VectorAttribute, the Vector contains events which are packed
         * three to a byte. We process one byte of the Vector at a time.
         */
        while (valen > 0) {
                if (skb_copy_bits(skb, *offset, &vaevents,
                                  sizeof(vaevents)) < 0)
                        return -1;
                *offset += sizeof(vaevents);

                /* Extract and process the first event. */
                vaevent = vaevents / (__MRP_VECATTR_EVENT_MAX *
                                      __MRP_VECATTR_EVENT_MAX);
                if (vaevent >= __MRP_VECATTR_EVENT_MAX) {
                        /* The byte is malformed; stop processing. */
                        return -1;
                }
                mrp_pdu_parse_vecattr_event(app, skb, vaevent);

                /* If present, extract and process the second event. */
                if (!--valen)
                        break;
                mrp_attrvalue_inc(mrp_cb(skb)->attrvalue,
                                  mrp_cb(skb)->mh->attrlen);
                vaevents %= (__MRP_VECATTR_EVENT_MAX *
                             __MRP_VECATTR_EVENT_MAX);
                vaevent = vaevents / __MRP_VECATTR_EVENT_MAX;
                mrp_pdu_parse_vecattr_event(app, skb, vaevent);

                /* If present, extract and process the third event. */
                if (!--valen)
                        break;
                mrp_attrvalue_inc(mrp_cb(skb)->attrvalue,
                                  mrp_cb(skb)->mh->attrlen);
                vaevents %= __MRP_VECATTR_EVENT_MAX;
                vaevent = vaevents;
                mrp_pdu_parse_vecattr_event(app, skb, vaevent);
        }
        return 0;
}

static int mrp_pdu_parse_msg(struct mrp_applicant *app, struct sk_buff *skb,
                             int *offset)
{
        struct mrp_msg_hdr _mh;

        mrp_cb(skb)->mh = skb_header_pointer(skb, *offset, sizeof(_mh), &_mh);
        if (!mrp_cb(skb)->mh)
                return -1;
        *offset += sizeof(_mh);

        if (mrp_cb(skb)->mh->attrtype == 0 ||
            mrp_cb(skb)->mh->attrtype > app->app->maxattr ||
            mrp_cb(skb)->mh->attrlen == 0)
                return -1;

        while (skb->len > *offset) {
                if (mrp_pdu_parse_end_mark(skb, offset) < 0)
                        break;
                if (mrp_pdu_parse_vecattr(app, skb, offset) < 0)
                        return -1;
        }
        return 0;
}

static int mrp_rcv(struct sk_buff *skb, struct net_device *dev,
                   struct packet_type *pt, struct net_device *orig_dev)
{
        struct mrp_application *appl = container_of(pt, struct mrp_application,
                                                    pkttype);
        struct mrp_port *port;
        struct mrp_applicant *app;
        struct mrp_pdu_hdr _ph;
        const struct mrp_pdu_hdr *ph;
        int offset = skb_network_offset(skb);

        /* If the interface is in promiscuous mode, drop the packet if
         * it was unicast to another host.
         */
        if (unlikely(skb->pkt_type == PACKET_OTHERHOST))
                goto out;
        skb = skb_share_check(skb, GFP_ATOMIC);
        if (unlikely(!skb))
                goto out;
        port = rcu_dereference(dev->mrp_port);
        if (unlikely(!port))
                goto out;
        app = rcu_dereference(port->applicants[appl->type]);
        if (unlikely(!app))
                goto out;

        ph = skb_header_pointer(skb, offset, sizeof(_ph), &_ph);
        if (!ph)
                goto out;
        offset += sizeof(_ph);

        if (ph->version != app->app->version)
                goto out;

        spin_lock(&app->lock);
        while (skb->len > offset) {
                if (mrp_pdu_parse_end_mark(skb, &offset) < 0)
                        break;
                if (mrp_pdu_parse_msg(app, skb, &offset) < 0)
                        break;
        }
        spin_unlock(&app->lock);
out:
        kfree_skb(skb);
        return 0;
}

static int mrp_init_port(struct net_device *dev)
{
        struct mrp_port *port;

        port = kzalloc(sizeof(*port), GFP_KERNEL);
        if (!port)
                return -ENOMEM;
        rcu_assign_pointer(dev->mrp_port, port);
        return 0;
}

static void mrp_release_port(struct net_device *dev)
{
        struct mrp_port *port = rtnl_dereference(dev->mrp_port);
        unsigned int i;

        for (i = 0; i <= MRP_APPLICATION_MAX; i++) {
                if (rtnl_dereference(port->applicants[i]))
                        return;
        }
        RCU_INIT_POINTER(dev->mrp_port, NULL);
        kfree_rcu(port, rcu);
}

int mrp_init_applicant(struct net_device *dev, struct mrp_application *appl)
{
        struct mrp_applicant *app;
        int err;

        ASSERT_RTNL();

        if (!rtnl_dereference(dev->mrp_port)) {
                err = mrp_init_port(dev);
                if (err < 0)
                        goto err1;
        }

        err = -ENOMEM;
        app = kzalloc(sizeof(*app), GFP_KERNEL);
        if (!app)
                goto err2;

        err = dev_mc_add(dev, appl->group_address);
        if (err < 0)
                goto err3;

        app->dev = dev;
        app->app = appl;
        app->mad = RB_ROOT;
        app->active = true;
        spin_lock_init(&app->lock);
        skb_queue_head_init(&app->queue);
        rcu_assign_pointer(dev->mrp_port->applicants[appl->type], app);
        timer_setup(&app->join_timer, mrp_join_timer, 0);
        mrp_join_timer_arm(app);
        timer_setup(&app->periodic_timer, mrp_periodic_timer, 0);
        mrp_periodic_timer_arm(app);
        return 0;

err3:
        kfree(app);
err2:
        mrp_release_port(dev);
err1:
        return err;
}
EXPORT_SYMBOL_GPL(mrp_init_applicant);

void mrp_uninit_applicant(struct net_device *dev, struct mrp_application *appl)
{
        struct mrp_port *port = rtnl_dereference(dev->mrp_port);
        struct mrp_applicant *app = rtnl_dereference(
                port->applicants[appl->type]);

        ASSERT_RTNL();

        RCU_INIT_POINTER(port->applicants[appl->type], NULL);

        spin_lock_bh(&app->lock);
        app->active = false;
        spin_unlock_bh(&app->lock);
        /* Delete timer and generate a final TX event to flush out
         * all pending messages before the applicant is gone.
         */
        timer_shutdown_sync(&app->join_timer);
        timer_shutdown_sync(&app->periodic_timer);

        spin_lock_bh(&app->lock);
        mrp_mad_event(app, MRP_EVENT_TX);
        mrp_attr_destroy_all(app);
        mrp_pdu_queue(app);
        spin_unlock_bh(&app->lock);

        mrp_queue_xmit(app);

        dev_mc_del(dev, appl->group_address);
        kfree_rcu(app, rcu);
        mrp_release_port(dev);
}
EXPORT_SYMBOL_GPL(mrp_uninit_applicant);

int mrp_register_application(struct mrp_application *appl)
{
        appl->pkttype.func = mrp_rcv;
        dev_add_pack(&appl->pkttype);
        return 0;
}
EXPORT_SYMBOL_GPL(mrp_register_application);

void mrp_unregister_application(struct mrp_application *appl)
{
        dev_remove_pack(&appl->pkttype);
}
EXPORT_SYMBOL_GPL(mrp_unregister_application);























   18 




    3 






   18 














   17 

   18 
   18 















   18 

   17 





   17 





    2 





   18 







   18 
   18 
   18 



   18 




   20 
   20 
   17 







   14 




   14 
    1 




   13 





    6 






    6 
    6 







    6 






    6 




    6 






    4 




    4 
    2 




    2 


























   20 









   20 


   20 



   14 






   14 
   14 


































































































































































































































   11 



    3 
    8 


    8 
    3 


   11 




   11 



   22 





   22 
   22 




   22 









   14 





   14 
   13 





    7 











    1 








    1 
    1 






    1 











    1 













































































   27 




























 1356 

























   16 













   14 



    2 


































   16 



   15 

    1 








    3 



    5 


    6 
    2 



    7 

    1 



















    9 


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
// SPDX-License-Identifier: GPL-2.0
/* Copyright 2011-2014 Autronica Fire and Security AS
 *
 * Author(s):
 *        2011-2014 Arvid Brodin, arvid.brodin@alten.se
 * This file contains device methods for creating, using and destroying
 * virtual HSR or PRP devices.
 */

#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/etherdevice.h>
#include <linux/rtnetlink.h>
#include <linux/pkt_sched.h>
#include "hsr_device.h"
#include "hsr_slave.h"
#include "hsr_framereg.h"
#include "hsr_main.h"
#include "hsr_forward.h"

static bool is_admin_up(struct net_device *dev)
{
        return dev && (dev->flags & IFF_UP);
}

static bool is_slave_up(struct net_device *dev)
{
        return dev && is_admin_up(dev) && netif_oper_up(dev);
}

static void hsr_set_operstate(struct hsr_port *master, bool has_carrier)
{
        struct net_device *dev = master->dev;

        if (!is_admin_up(dev)) {
                netif_set_operstate(dev, IF_OPER_DOWN);
                return;
        }

        if (has_carrier)
                netif_set_operstate(dev, IF_OPER_UP);
        else
                netif_set_operstate(dev, IF_OPER_LOWERLAYERDOWN);
}

static bool hsr_check_carrier(struct hsr_port *master)
{
        struct hsr_port *port;

        ASSERT_RTNL();

        hsr_for_each_port(master->hsr, port) {
                if (port->type != HSR_PT_MASTER && is_slave_up(port->dev)) {
                        netif_carrier_on(master->dev);
                        return true;
                }
        }

        netif_carrier_off(master->dev);

        return false;
}

static void hsr_check_announce(struct net_device *hsr_dev)
{
        struct hsr_priv *hsr;

        hsr = netdev_priv(hsr_dev);
        if (netif_running(hsr_dev) && netif_oper_up(hsr_dev)) {
                /* Enable announce timer and start sending supervisory frames */
                if (!timer_pending(&hsr->announce_timer)) {
                        hsr->announce_count = 0;
                        mod_timer(&hsr->announce_timer, jiffies +
                                  msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL));
                }

                if (hsr->redbox && !timer_pending(&hsr->announce_proxy_timer))
                        mod_timer(&hsr->announce_proxy_timer, jiffies +
                                  msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL) / 2);
        } else {
                /* Deactivate the announce timer  */
                timer_delete(&hsr->announce_timer);
                if (hsr->redbox)
                        timer_delete(&hsr->announce_proxy_timer);
        }
}

void hsr_check_carrier_and_operstate(struct hsr_priv *hsr)
{
        struct hsr_port *master;
        bool has_carrier;

        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        /* netif_stacked_transfer_operstate() cannot be used here since
         * it doesn't set IF_OPER_LOWERLAYERDOWN (?)
         */
        has_carrier = hsr_check_carrier(master);
        hsr_set_operstate(master, has_carrier);
        hsr_check_announce(master->dev);
}

int hsr_get_max_mtu(struct hsr_priv *hsr)
{
        unsigned int mtu_max;
        struct hsr_port *port;

        mtu_max = ETH_DATA_LEN;
        hsr_for_each_port(hsr, port)
                if (port->type != HSR_PT_MASTER)
                        mtu_max = min(port->dev->mtu, mtu_max);

        if (mtu_max < HSR_HLEN)
                return 0;
        return mtu_max - HSR_HLEN;
}

static int hsr_dev_change_mtu(struct net_device *dev, int new_mtu)
{
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        if (new_mtu > hsr_get_max_mtu(hsr)) {
                netdev_info(dev, "A HSR master's MTU cannot be greater than the smallest MTU of its slaves minus the HSR Tag length (%d octets).\n",
                            HSR_HLEN);
                return -EINVAL;
        }

        WRITE_ONCE(dev->mtu, new_mtu);

        return 0;
}

static int hsr_dev_open(struct net_device *dev)
{
        struct hsr_priv *hsr;
        struct hsr_port *port;
        const char *designation = NULL;

        hsr = netdev_priv(dev);

        hsr_for_each_port(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                        designation = "Slave A";
                        break;
                case HSR_PT_SLAVE_B:
                        designation = "Slave B";
                        break;
                case HSR_PT_INTERLINK:
                        designation = "Interlink";
                        break;
                default:
                        designation = "Unknown";
                }
                if (!is_slave_up(port->dev))
                        netdev_warn(dev, "%s (%s) is not up; please bring it up to get a fully working HSR network\n",
                                    designation, port->dev->name);
        }

        if (!designation)
                netdev_warn(dev, "No slave devices configured\n");

        return 0;
}

static int hsr_dev_close(struct net_device *dev)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);
        hsr_for_each_port(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        dev_uc_unsync(port->dev, dev);
                        dev_mc_unsync(port->dev, dev);
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static netdev_features_t hsr_features_recompute(struct hsr_priv *hsr,
                                                netdev_features_t features)
{
        netdev_features_t mask;
        struct hsr_port *port;

        mask = features;

        /* Mask out all features that, if supported by one device, should be
         * enabled for all devices (see NETIF_F_ONE_FOR_ALL).
         *
         * Anything that's off in mask will not be enabled - so only things
         * that were in features originally, and also is in NETIF_F_ONE_FOR_ALL,
         * may become enabled.
         */
        features &= ~NETIF_F_ONE_FOR_ALL;
        hsr_for_each_port(hsr, port)
                features = netdev_increment_features(features,
                                                     port->dev->features,
                                                     mask);

        return features;
}

static netdev_features_t hsr_fix_features(struct net_device *dev,
                                          netdev_features_t features)
{
        struct hsr_priv *hsr = netdev_priv(dev);

        return hsr_features_recompute(hsr, features);
}

static netdev_tx_t hsr_dev_xmit(struct sk_buff *skb, struct net_device *dev)
{
        struct hsr_priv *hsr = netdev_priv(dev);
        struct hsr_port *master;

        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        if (master) {
                skb->dev = master->dev;
                skb_reset_mac_header(skb);
                skb_reset_mac_len(skb);
                spin_lock_bh(&hsr->seqnr_lock);
                hsr_forward_skb(skb, master);
                spin_unlock_bh(&hsr->seqnr_lock);
        } else {
                dev_core_stats_tx_dropped_inc(dev);
                dev_kfree_skb_any(skb);
        }
        return NETDEV_TX_OK;
}

static const struct header_ops hsr_header_ops = {
        .create         = eth_header,
        .parse         = eth_header_parse,
};

static struct sk_buff *hsr_init_skb(struct hsr_port *master, int extra)
{
        struct hsr_priv *hsr = master->hsr;
        struct sk_buff *skb;
        int hlen, tlen;
        int len;

        hlen = LL_RESERVED_SPACE(master->dev);
        tlen = master->dev->needed_tailroom;
        len = sizeof(struct hsr_sup_tag) + sizeof(struct hsr_sup_payload);
        /* skb size is same for PRP/HSR frames, only difference
         * being, for PRP it is a trailer and for HSR it is a
         * header.
         * RedBox might use @extra more bytes.
         */
        skb = dev_alloc_skb(len + extra + hlen + tlen);

        if (!skb)
                return skb;

        skb_reserve(skb, hlen);
        skb->dev = master->dev;
        skb->priority = TC_PRIO_CONTROL;

        skb_reset_network_header(skb);
        skb_reset_transport_header(skb);
        if (dev_hard_header(skb, skb->dev, ETH_P_PRP,
                            hsr->sup_multicast_addr,
                            skb->dev->dev_addr, skb->len) <= 0)
                goto out;

        skb_reset_mac_header(skb);
        skb_reset_mac_len(skb);

        return skb;
out:
        kfree_skb(skb);

        return NULL;
}

static void send_hsr_supervision_frame(struct hsr_port *port,
                                       unsigned long *interval,
                                       const unsigned char *addr)
{
        struct hsr_priv *hsr = port->hsr;
        __u8 type = HSR_TLV_LIFE_CHECK;
        struct hsr_sup_payload *hsr_sp;
        struct hsr_sup_tlv *hsr_stlv;
        struct hsr_sup_tag *hsr_stag;
        struct sk_buff *skb;
        int extra = 0;

        *interval = msecs_to_jiffies(HSR_LIFE_CHECK_INTERVAL);
        if (hsr->announce_count < 3 && hsr->prot_version == 0) {
                type = HSR_TLV_ANNOUNCE;
                *interval = msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL);
                hsr->announce_count++;
        }

        if (hsr->redbox)
                extra = sizeof(struct hsr_sup_tlv) +
                        sizeof(struct hsr_sup_payload);

        skb = hsr_init_skb(port, extra);
        if (!skb) {
                netdev_warn_once(port->dev, "HSR: Could not send supervision frame\n");
                return;
        }

        hsr_stag = skb_put(skb, sizeof(struct hsr_sup_tag));
        set_hsr_stag_path(hsr_stag, (hsr->prot_version ? 0x0 : 0xf));
        set_hsr_stag_HSR_ver(hsr_stag, hsr->prot_version);

        /* From HSRv1 on we have separate supervision sequence numbers. */
        spin_lock_bh(&hsr->seqnr_lock);
        if (hsr->prot_version > 0) {
                hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr);
                hsr->sup_sequence_nr++;
        } else {
                hsr_stag->sequence_nr = htons(hsr->sequence_nr);
                hsr->sequence_nr++;
        }

        hsr_stag->tlv.HSR_TLV_type = type;
        /* TODO: Why 12 in HSRv0? */
        hsr_stag->tlv.HSR_TLV_length = hsr->prot_version ?
                                sizeof(struct hsr_sup_payload) : 12;

        /* Payload: MacAddressA / SAN MAC from ProxyNodeTable */
        hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
        ether_addr_copy(hsr_sp->macaddress_A, addr);

        if (hsr->redbox &&
            hsr_is_node_in_db(&hsr->proxy_node_db, addr)) {
                hsr_stlv = skb_put(skb, sizeof(struct hsr_sup_tlv));
                hsr_stlv->HSR_TLV_type = PRP_TLV_REDBOX_MAC;
                hsr_stlv->HSR_TLV_length = sizeof(struct hsr_sup_payload);

                /* Payload: MacAddressRedBox */
                hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
                ether_addr_copy(hsr_sp->macaddress_A, hsr->macaddress_redbox);
        }

        if (skb_put_padto(skb, ETH_ZLEN)) {
                spin_unlock_bh(&hsr->seqnr_lock);
                return;
        }

        hsr_forward_skb(skb, port);
        spin_unlock_bh(&hsr->seqnr_lock);
        return;
}

static void send_prp_supervision_frame(struct hsr_port *master,
                                       unsigned long *interval,
                                       const unsigned char *addr)
{
        struct hsr_priv *hsr = master->hsr;
        struct hsr_sup_payload *hsr_sp;
        struct hsr_sup_tag *hsr_stag;
        struct sk_buff *skb;

        skb = hsr_init_skb(master, 0);
        if (!skb) {
                netdev_warn_once(master->dev, "PRP: Could not send supervision frame\n");
                return;
        }

        *interval = msecs_to_jiffies(HSR_LIFE_CHECK_INTERVAL);
        hsr_stag = skb_put(skb, sizeof(struct hsr_sup_tag));
        set_hsr_stag_path(hsr_stag, (hsr->prot_version ? 0x0 : 0xf));
        set_hsr_stag_HSR_ver(hsr_stag, (hsr->prot_version ? 1 : 0));

        /* From HSRv1 on we have separate supervision sequence numbers. */
        spin_lock_bh(&hsr->seqnr_lock);
        hsr_stag->sequence_nr = htons(hsr->sup_sequence_nr);
        hsr->sup_sequence_nr++;
        hsr_stag->tlv.HSR_TLV_type = PRP_TLV_LIFE_CHECK_DD;
        hsr_stag->tlv.HSR_TLV_length = sizeof(struct hsr_sup_payload);

        /* Payload: MacAddressA */
        hsr_sp = skb_put(skb, sizeof(struct hsr_sup_payload));
        ether_addr_copy(hsr_sp->macaddress_A, master->dev->dev_addr);

        if (skb_put_padto(skb, ETH_ZLEN)) {
                spin_unlock_bh(&hsr->seqnr_lock);
                return;
        }

        hsr_forward_skb(skb, master);
        spin_unlock_bh(&hsr->seqnr_lock);
}

/* Announce (supervision frame) timer function
 */
static void hsr_announce(struct timer_list *t)
{
        struct hsr_priv *hsr;
        struct hsr_port *master;
        unsigned long interval;

        hsr = timer_container_of(hsr, t, announce_timer);

        rcu_read_lock();
        master = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        hsr->proto_ops->send_sv_frame(master, &interval, master->dev->dev_addr);

        if (is_admin_up(master->dev))
                mod_timer(&hsr->announce_timer, jiffies + interval);

        rcu_read_unlock();
}

/* Announce (supervision frame) timer function for RedBox
 */
static void hsr_proxy_announce(struct timer_list *t)
{
        struct hsr_priv *hsr = timer_container_of(hsr, t,
                                                  announce_proxy_timer);
        struct hsr_port *interlink;
        unsigned long interval = 0;
        struct hsr_node *node;

        rcu_read_lock();
        /* RedBOX sends supervisory frames to HSR network with MAC addresses
         * of SAN nodes stored in ProxyNodeTable.
         */
        interlink = hsr_port_get_hsr(hsr, HSR_PT_INTERLINK);
        if (!interlink)
                goto done;

        list_for_each_entry_rcu(node, &hsr->proxy_node_db, mac_list) {
                if (hsr_addr_is_redbox(hsr, node->macaddress_A))
                        continue;
                hsr->proto_ops->send_sv_frame(interlink, &interval,
                                              node->macaddress_A);
        }

        if (is_admin_up(interlink->dev)) {
                if (!interval)
                        interval = msecs_to_jiffies(HSR_ANNOUNCE_INTERVAL);

                mod_timer(&hsr->announce_proxy_timer, jiffies + interval);
        }

done:
        rcu_read_unlock();
}

void hsr_del_ports(struct hsr_priv *hsr)
{
        struct hsr_port *port;

        port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_A);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_SLAVE_B);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_INTERLINK);
        if (port)
                hsr_del_port(port);

        port = hsr_port_get_hsr(hsr, HSR_PT_MASTER);
        if (port)
                hsr_del_port(port);
}

static void hsr_set_rx_mode(struct net_device *dev)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        dev_mc_sync_multiple(port->dev, dev);
                        dev_uc_sync_multiple(port->dev, dev);
                        break;
                default:
                        break;
                }
        }
}

static void hsr_change_rx_flags(struct net_device *dev, int change)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port(hsr, port) {
                if (port->type == HSR_PT_MASTER)
                        continue;
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        if (change & IFF_ALLMULTI)
                                dev_set_allmulti(port->dev,
                                                 dev->flags &
                                                 IFF_ALLMULTI ? 1 : -1);
                        break;
                default:
                        break;
                }
        }
}

static int hsr_ndo_vlan_rx_add_vid(struct net_device *dev,
                                   __be16 proto, u16 vid)
{
        bool is_slave_a_added = false;
        bool is_slave_b_added = false;
        struct hsr_port *port;
        struct hsr_priv *hsr;
        int ret = 0;

        hsr = netdev_priv(dev);

        hsr_for_each_port(hsr, port) {
                if (port->type == HSR_PT_MASTER ||
                    port->type == HSR_PT_INTERLINK)
                        continue;

                ret = vlan_vid_add(port->dev, proto, vid);
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                        if (ret) {
                                /* clean up Slave-B */
                                netdev_err(dev, "add vid failed for Slave-A\n");
                                if (is_slave_b_added)
                                        vlan_vid_del(port->dev, proto, vid);
                                return ret;
                        }

                        is_slave_a_added = true;
                        break;

                case HSR_PT_SLAVE_B:
                        if (ret) {
                                /* clean up Slave-A */
                                netdev_err(dev, "add vid failed for Slave-B\n");
                                if (is_slave_a_added)
                                        vlan_vid_del(port->dev, proto, vid);
                                return ret;
                        }

                        is_slave_b_added = true;
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static int hsr_ndo_vlan_rx_kill_vid(struct net_device *dev,
                                    __be16 proto, u16 vid)
{
        struct hsr_port *port;
        struct hsr_priv *hsr;

        hsr = netdev_priv(dev);

        hsr_for_each_port(hsr, port) {
                switch (port->type) {
                case HSR_PT_SLAVE_A:
                case HSR_PT_SLAVE_B:
                        vlan_vid_del(port->dev, proto, vid);
                        break;
                default:
                        break;
                }
        }

        return 0;
}

static const struct net_device_ops hsr_device_ops = {
        .ndo_change_mtu = hsr_dev_change_mtu,
        .ndo_open = hsr_dev_open,
        .ndo_stop = hsr_dev_close,
        .ndo_start_xmit = hsr_dev_xmit,
        .ndo_change_rx_flags = hsr_change_rx_flags,
        .ndo_fix_features = hsr_fix_features,
        .ndo_set_rx_mode = hsr_set_rx_mode,
        .ndo_vlan_rx_add_vid = hsr_ndo_vlan_rx_add_vid,
        .ndo_vlan_rx_kill_vid = hsr_ndo_vlan_rx_kill_vid,
};

static const struct device_type hsr_type = {
        .name = "hsr",
};

static struct hsr_proto_ops hsr_ops = {
        .send_sv_frame = send_hsr_supervision_frame,
        .create_tagged_frame = hsr_create_tagged_frame,
        .get_untagged_frame = hsr_get_untagged_frame,
        .drop_frame = hsr_drop_frame,
        .fill_frame_info = hsr_fill_frame_info,
        .invalid_dan_ingress_frame = hsr_invalid_dan_ingress_frame,
        .register_frame_out = hsr_register_frame_out,
};

static struct hsr_proto_ops prp_ops = {
        .send_sv_frame = send_prp_supervision_frame,
        .create_tagged_frame = prp_create_tagged_frame,
        .get_untagged_frame = prp_get_untagged_frame,
        .drop_frame = prp_drop_frame,
        .fill_frame_info = prp_fill_frame_info,
        .handle_san_frame = prp_handle_san_frame,
        .update_san_info = prp_update_san_info,
        .register_frame_out = prp_register_frame_out,
};

void hsr_dev_setup(struct net_device *dev)
{
        eth_hw_addr_random(dev);

        ether_setup(dev);
        dev->min_mtu = 0;
        dev->header_ops = &hsr_header_ops;
        dev->netdev_ops = &hsr_device_ops;
        SET_NETDEV_DEVTYPE(dev, &hsr_type);
        dev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL;
        /* Prevent recursive tx locking */
        dev->lltx = true;
        /* Not sure about this. Taken from bridge code. netdevice.h says
         * it means "Does not change network namespaces".
         */
        dev->netns_immutable = true;

        dev->needs_free_netdev = true;

        dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HIGHDMA |
                           NETIF_F_GSO_MASK | NETIF_F_HW_CSUM |
                           NETIF_F_HW_VLAN_CTAG_TX |
                           NETIF_F_HW_VLAN_CTAG_FILTER;

        dev->features = dev->hw_features;
}

/* Return true if dev is a HSR master; return false otherwise.
 */
bool is_hsr_master(struct net_device *dev)
{
        return (dev->netdev_ops->ndo_start_xmit == hsr_dev_xmit);
}
EXPORT_SYMBOL(is_hsr_master);

struct net_device *hsr_get_port_ndev(struct net_device *ndev,
                                     enum hsr_port_type pt)
{
        struct hsr_priv *hsr = netdev_priv(ndev);
        struct hsr_port *port;

        hsr_for_each_port(hsr, port)
                if (port->type == pt)
                        return port->dev;
        return NULL;
}
EXPORT_SYMBOL(hsr_get_port_ndev);

/* Default multicast address for HSR Supervision frames */
static const unsigned char def_multicast_addr[ETH_ALEN] __aligned(2) = {
        0x01, 0x15, 0x4e, 0x00, 0x01, 0x00
};

int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2],
                     struct net_device *interlink, unsigned char multicast_spec,
                     u8 protocol_version, struct netlink_ext_ack *extack)
{
        bool unregister = false;
        struct hsr_priv *hsr;
        int res;

        hsr = netdev_priv(hsr_dev);
        INIT_LIST_HEAD(&hsr->ports);
        INIT_LIST_HEAD(&hsr->node_db);
        INIT_LIST_HEAD(&hsr->proxy_node_db);
        spin_lock_init(&hsr->list_lock);

        eth_hw_addr_set(hsr_dev, slave[0]->dev_addr);

        /* initialize protocol specific functions */
        if (protocol_version == PRP_V1) {
                /* For PRP, lan_id has most significant 3 bits holding
                 * the net_id of PRP_LAN_ID
                 */
                hsr->net_id = PRP_LAN_ID << 1;
                hsr->proto_ops = &prp_ops;
        } else {
                hsr->proto_ops = &hsr_ops;
        }

        /* Make sure we recognize frames from ourselves in hsr_rcv() */
        res = hsr_create_self_node(hsr, hsr_dev->dev_addr,
                                   slave[1]->dev_addr);
        if (res < 0)
                return res;

        spin_lock_init(&hsr->seqnr_lock);
        /* Overflow soon to find bugs easier: */
        hsr->sequence_nr = HSR_SEQNR_START;
        hsr->sup_sequence_nr = HSR_SUP_SEQNR_START;

        timer_setup(&hsr->announce_timer, hsr_announce, 0);
        timer_setup(&hsr->prune_timer, hsr_prune_nodes, 0);
        timer_setup(&hsr->prune_proxy_timer, hsr_prune_proxy_nodes, 0);
        timer_setup(&hsr->announce_proxy_timer, hsr_proxy_announce, 0);

        ether_addr_copy(hsr->sup_multicast_addr, def_multicast_addr);
        hsr->sup_multicast_addr[ETH_ALEN - 1] = multicast_spec;

        hsr->prot_version = protocol_version;

        /* Make sure the 1st call to netif_carrier_on() gets through */
        netif_carrier_off(hsr_dev);

        res = hsr_add_port(hsr, hsr_dev, HSR_PT_MASTER, extack);
        if (res)
                goto err_add_master;

        /* HSR forwarding offload supported in lower device? */
        if ((slave[0]->features & NETIF_F_HW_HSR_FWD) &&
            (slave[1]->features & NETIF_F_HW_HSR_FWD))
                hsr->fwd_offloaded = true;

        if ((slave[0]->features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
            (slave[1]->features & NETIF_F_HW_VLAN_CTAG_FILTER))
                hsr_dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;

        res = register_netdevice(hsr_dev);
        if (res)
                goto err_unregister;

        unregister = true;

        res = hsr_add_port(hsr, slave[0], HSR_PT_SLAVE_A, extack);
        if (res)
                goto err_unregister;

        res = hsr_add_port(hsr, slave[1], HSR_PT_SLAVE_B, extack);
        if (res)
                goto err_unregister;

        if (protocol_version == PRP_V1) {
                eth_hw_addr_set(slave[1], slave[0]->dev_addr);
                call_netdevice_notifiers(NETDEV_CHANGEADDR, slave[1]);
        }

        if (interlink) {
                res = hsr_add_port(hsr, interlink, HSR_PT_INTERLINK, extack);
                if (res)
                        goto err_unregister;

                hsr->redbox = true;
                ether_addr_copy(hsr->macaddress_redbox, interlink->dev_addr);
                mod_timer(&hsr->prune_proxy_timer,
                          jiffies + msecs_to_jiffies(PRUNE_PROXY_PERIOD));
        }

        hsr_debugfs_init(hsr, hsr_dev);
        mod_timer(&hsr->prune_timer, jiffies + msecs_to_jiffies(PRUNE_PERIOD));

        return 0;

err_unregister:
        hsr_del_ports(hsr);
err_add_master:
        hsr_del_self_node(hsr);

        if (unregister)
                unregister_netdevice(hsr_dev);
        return res;
}





































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    3 










  146 





















    3 

















    3 

















   13 
   59 




















    3 

















    3 


















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  V4L2 sub-device support header.
 *
 *  Copyright (C) 2008  Hans Verkuil <hverkuil@xs4all.nl>
 */

#ifndef _V4L2_SUBDEV_H
#define _V4L2_SUBDEV_H

#include <linux/types.h>
#include <linux/v4l2-subdev.h>
#include <media/media-entity.h>
#include <media/v4l2-async.h>
#include <media/v4l2-common.h>
#include <media/v4l2-dev.h>
#include <media/v4l2-fh.h>
#include <media/v4l2-mediabus.h>

/* generic v4l2_device notify callback notification values */
#define V4L2_SUBDEV_IR_RX_NOTIFY                _IOW('v', 0, u32)
#define V4L2_SUBDEV_IR_RX_FIFO_SERVICE_REQ        0x00000001
#define V4L2_SUBDEV_IR_RX_END_OF_RX_DETECTED        0x00000002
#define V4L2_SUBDEV_IR_RX_HW_FIFO_OVERRUN        0x00000004
#define V4L2_SUBDEV_IR_RX_SW_FIFO_OVERRUN        0x00000008

#define V4L2_SUBDEV_IR_TX_NOTIFY                _IOW('v', 1, u32)
#define V4L2_SUBDEV_IR_TX_FIFO_SERVICE_REQ        0x00000001

#define        V4L2_DEVICE_NOTIFY_EVENT                _IOW('v', 2, struct v4l2_event)

struct v4l2_device;
struct v4l2_ctrl_handler;
struct v4l2_event;
struct v4l2_event_subscription;
struct v4l2_fh;
struct v4l2_subdev;
struct v4l2_subdev_fh;
struct tuner_setup;
struct v4l2_mbus_frame_desc;
struct led_classdev;

/**
 * struct v4l2_decode_vbi_line - used to decode_vbi_line
 *
 * @is_second_field: Set to 0 for the first (odd) field;
 *        set to 1 for the second (even) field.
 * @p: Pointer to the sliced VBI data from the decoder. On exit, points to
 *        the start of the payload.
 * @line: Line number of the sliced VBI data (1-23)
 * @type: VBI service type (V4L2_SLICED_*). 0 if no service found
 */
struct v4l2_decode_vbi_line {
        u32 is_second_field;
        u8 *p;
        u32 line;
        u32 type;
};

/*
 * Sub-devices are devices that are connected somehow to the main bridge
 * device. These devices are usually audio/video muxers/encoders/decoders or
 * sensors and webcam controllers.
 *
 * Usually these devices are controlled through an i2c bus, but other buses
 * may also be used.
 *
 * The v4l2_subdev struct provides a way of accessing these devices in a
 * generic manner. Most operations that these sub-devices support fall in
 * a few categories: core ops, audio ops, video ops and tuner ops.
 *
 * More categories can be added if needed, although this should remain a
 * limited set (no more than approx. 8 categories).
 *
 * Each category has its own set of ops that subdev drivers can implement.
 *
 * A subdev driver can leave the pointer to the category ops NULL if
 * it does not implement them (e.g. an audio subdev will generally not
 * implement the video category ops). The exception is the core category:
 * this must always be present.
 *
 * These ops are all used internally so it is no problem to change, remove
 * or add ops or move ops from one to another category. Currently these
 * ops are based on the original ioctls, but since ops are not limited to
 * one argument there is room for improvement here once all i2c subdev
 * drivers are converted to use these ops.
 */

/*
 * Core ops: it is highly recommended to implement at least these ops:
 *
 * log_status
 * g_register
 * s_register
 *
 * This provides basic debugging support.
 *
 * The ioctl ops is meant for generic ioctl-like commands. Depending on
 * the use-case it might be better to use subdev-specific ops (currently
 * not yet implemented) since ops provide proper type-checking.
 */

/**
 * enum v4l2_subdev_io_pin_bits - Subdevice external IO pin configuration
 *        bits
 *
 * @V4L2_SUBDEV_IO_PIN_DISABLE: disables a pin config. ENABLE assumed.
 * @V4L2_SUBDEV_IO_PIN_OUTPUT: set it if pin is an output.
 * @V4L2_SUBDEV_IO_PIN_INPUT: set it if pin is an input.
 * @V4L2_SUBDEV_IO_PIN_SET_VALUE: to set the output value via
 *                                  &struct v4l2_subdev_io_pin_config->value.
 * @V4L2_SUBDEV_IO_PIN_ACTIVE_LOW: pin active is bit 0.
 *                                   Otherwise, ACTIVE HIGH is assumed.
 */
enum v4l2_subdev_io_pin_bits {
        V4L2_SUBDEV_IO_PIN_DISABLE        = 0,
        V4L2_SUBDEV_IO_PIN_OUTPUT        = 1,
        V4L2_SUBDEV_IO_PIN_INPUT        = 2,
        V4L2_SUBDEV_IO_PIN_SET_VALUE        = 3,
        V4L2_SUBDEV_IO_PIN_ACTIVE_LOW        = 4,
};

/**
 * struct v4l2_subdev_io_pin_config - Subdevice external IO pin configuration
 *
 * @flags: bitmask with flags for this pin's config, whose bits are defined by
 *           &enum v4l2_subdev_io_pin_bits.
 * @pin: Chip external IO pin to configure
 * @function: Internal signal pad/function to route to IO pin
 * @value: Initial value for pin - e.g. GPIO output value
 * @strength: Pin drive strength
 */
struct v4l2_subdev_io_pin_config {
        u32 flags;
        u8 pin;
        u8 function;
        u8 value;
        u8 strength;
};

/**
 * struct v4l2_subdev_core_ops - Define core ops callbacks for subdevs
 *
 * @log_status: callback for VIDIOC_LOG_STATUS() ioctl handler code.
 *
 * @s_io_pin_config: configure one or more chip I/O pins for chips that
 *        multiplex different internal signal pads out to IO pins.  This function
 *        takes a pointer to an array of 'n' pin configuration entries, one for
 *        each pin being configured.  This function could be called at times
 *        other than just subdevice initialization.
 *
 * @init: initialize the sensor registers to some sort of reasonable default
 *        values. Do not use for new drivers and should be removed in existing
 *        drivers.
 *
 * @load_fw: load firmware.
 *
 * @reset: generic reset command. The argument selects which subsystems to
 *        reset. Passing 0 will always reset the whole chip. Do not use for new
 *        drivers without discussing this first on the linux-media mailinglist.
 *        There should be no reason normally to reset a device.
 *
 * @s_gpio: set GPIO pins. Very simple right now, might need to be extended with
 *        a direction argument if needed.
 *
 * @command: called by in-kernel drivers in order to call functions internal
 *           to subdev drivers driver that have a separate callback.
 *
 * @ioctl: called at the end of ioctl() syscall handler at the V4L2 core.
 *           used to provide support for private ioctls used on the driver.
 *
 * @compat_ioctl32: called when a 32 bits application uses a 64 bits Kernel,
 *                    in order to fix data passed from/to userspace.
 *
 * @g_register: callback for VIDIOC_DBG_G_REGISTER() ioctl handler code.
 *
 * @s_register: callback for VIDIOC_DBG_S_REGISTER() ioctl handler code.
 *
 * @s_power: puts subdevice in power saving mode (on == 0) or normal operation
 *        mode (on == 1). DEPRECATED. See
 *        Documentation/driver-api/media/camera-sensor.rst . pre_streamon and
 *        post_streamoff callbacks can be used for e.g. setting the bus to LP-11
 *        mode before s_stream is called.
 *
 * @interrupt_service_routine: Called by the bridge chip's interrupt service
 *        handler, when an interrupt status has be raised due to this subdev,
 *        so that this subdev can handle the details.  It may schedule work to be
 *        performed later.  It must not sleep. **Called from an IRQ context**.
 *
 * @subscribe_event: used by the drivers to request the control framework that
 *                     for it to be warned when the value of a control changes.
 *
 * @unsubscribe_event: remove event subscription from the control framework.
 */
struct v4l2_subdev_core_ops {
        int (*log_status)(struct v4l2_subdev *sd);
        int (*s_io_pin_config)(struct v4l2_subdev *sd, size_t n,
                                      struct v4l2_subdev_io_pin_config *pincfg);
        int (*init)(struct v4l2_subdev *sd, u32 val);
        int (*load_fw)(struct v4l2_subdev *sd);
        int (*reset)(struct v4l2_subdev *sd, u32 val);
        int (*s_gpio)(struct v4l2_subdev *sd, u32 val);
        long (*command)(struct v4l2_subdev *sd, unsigned int cmd, void *arg);
        long (*ioctl)(struct v4l2_subdev *sd, unsigned int cmd, void *arg);
#ifdef CONFIG_COMPAT
        long (*compat_ioctl32)(struct v4l2_subdev *sd, unsigned int cmd,
                               unsigned long arg);
#endif
#ifdef CONFIG_VIDEO_ADV_DEBUG
        int (*g_register)(struct v4l2_subdev *sd, struct v4l2_dbg_register *reg);
        int (*s_register)(struct v4l2_subdev *sd, const struct v4l2_dbg_register *reg);
#endif
        int (*s_power)(struct v4l2_subdev *sd, int on);
        int (*interrupt_service_routine)(struct v4l2_subdev *sd,
                                                u32 status, bool *handled);
        int (*subscribe_event)(struct v4l2_subdev *sd, struct v4l2_fh *fh,
                               struct v4l2_event_subscription *sub);
        int (*unsubscribe_event)(struct v4l2_subdev *sd, struct v4l2_fh *fh,
                                 struct v4l2_event_subscription *sub);
};

/**
 * struct v4l2_subdev_tuner_ops - Callbacks used when v4l device was opened
 *        in radio mode.
 *
 * @standby: puts the tuner in standby mode. It will be woken up
 *             automatically the next time it is used.
 *
 * @s_radio: callback that switches the tuner to radio mode.
 *             drivers should explicitly call it when a tuner ops should
 *             operate on radio mode, before being able to handle it.
 *             Used on devices that have both AM/FM radio receiver and TV.
 *
 * @s_frequency: callback for VIDIOC_S_FREQUENCY() ioctl handler code.
 *
 * @g_frequency: callback for VIDIOC_G_FREQUENCY() ioctl handler code.
 *                 freq->type must be filled in. Normally done by video_ioctl2()
 *                 or the bridge driver.
 *
 * @enum_freq_bands: callback for VIDIOC_ENUM_FREQ_BANDS() ioctl handler code.
 *
 * @g_tuner: callback for VIDIOC_G_TUNER() ioctl handler code.
 *
 * @s_tuner: callback for VIDIOC_S_TUNER() ioctl handler code. @vt->type must be
 *             filled in. Normally done by video_ioctl2 or the
 *             bridge driver.
 *
 * @g_modulator: callback for VIDIOC_G_MODULATOR() ioctl handler code.
 *
 * @s_modulator: callback for VIDIOC_S_MODULATOR() ioctl handler code.
 *
 * @s_type_addr: sets tuner type and its I2C addr.
 *
 * @s_config: sets tda9887 specific stuff, like port1, port2 and qss
 *
 * .. note::
 *
 *        On devices that have both AM/FM and TV, it is up to the driver
 *        to explicitly call s_radio when the tuner should be switched to
 *        radio mode, before handling other &struct v4l2_subdev_tuner_ops
 *        that would require it. An example of such usage is::
 *
 *          static void s_frequency(void *priv, const struct v4l2_frequency *f)
 *          {
 *                ...
 *                if (f.type == V4L2_TUNER_RADIO)
 *                        v4l2_device_call_all(v4l2_dev, 0, tuner, s_radio);
 *                ...
 *                v4l2_device_call_all(v4l2_dev, 0, tuner, s_frequency);
 *          }
 */
struct v4l2_subdev_tuner_ops {
        int (*standby)(struct v4l2_subdev *sd);
        int (*s_radio)(struct v4l2_subdev *sd);
        int (*s_frequency)(struct v4l2_subdev *sd, const struct v4l2_frequency *freq);
        int (*g_frequency)(struct v4l2_subdev *sd, struct v4l2_frequency *freq);
        int (*enum_freq_bands)(struct v4l2_subdev *sd, struct v4l2_frequency_band *band);
        int (*g_tuner)(struct v4l2_subdev *sd, struct v4l2_tuner *vt);
        int (*s_tuner)(struct v4l2_subdev *sd, const struct v4l2_tuner *vt);
        int (*g_modulator)(struct v4l2_subdev *sd, struct v4l2_modulator *vm);
        int (*s_modulator)(struct v4l2_subdev *sd, const struct v4l2_modulator *vm);
        int (*s_type_addr)(struct v4l2_subdev *sd, struct tuner_setup *type);
        int (*s_config)(struct v4l2_subdev *sd, const struct v4l2_priv_tun_config *config);
};

/**
 * struct v4l2_subdev_audio_ops - Callbacks used for audio-related settings
 *
 * @s_clock_freq: set the frequency (in Hz) of the audio clock output.
 *        Used to slave an audio processor to the video decoder, ensuring that
 *        audio and video remain synchronized. Usual values for the frequency
 *        are 48000, 44100 or 32000 Hz. If the frequency is not supported, then
 *        -EINVAL is returned.
 *
 * @s_i2s_clock_freq: sets I2S speed in bps. This is used to provide a standard
 *        way to select I2S clock used by driving digital audio streams at some
 *        board designs. Usual values for the frequency are 1024000 and 2048000.
 *        If the frequency is not supported, then %-EINVAL is returned.
 *
 * @s_routing: used to define the input and/or output pins of an audio chip,
 *        and any additional configuration data.
 *        Never attempt to use user-level input IDs (e.g. Composite, S-Video,
 *        Tuner) at this level. An i2c device shouldn't know about whether an
 *        input pin is connected to a Composite connector, become on another
 *        board or platform it might be connected to something else entirely.
 *        The calling driver is responsible for mapping a user-level input to
 *        the right pins on the i2c device.
 *
 * @s_stream: used to notify the audio code that stream will start or has
 *        stopped.
 */
struct v4l2_subdev_audio_ops {
        int (*s_clock_freq)(struct v4l2_subdev *sd, u32 freq);
        int (*s_i2s_clock_freq)(struct v4l2_subdev *sd, u32 freq);
        int (*s_routing)(struct v4l2_subdev *sd, u32 input, u32 output, u32 config);
        int (*s_stream)(struct v4l2_subdev *sd, int enable);
};

/**
 * struct v4l2_mbus_frame_desc_entry_csi2
 *
 * @vc: CSI-2 virtual channel
 * @dt: CSI-2 data type ID
 */
struct v4l2_mbus_frame_desc_entry_csi2 {
        u8 vc;
        u8 dt;
};

/**
 * enum v4l2_mbus_frame_desc_flags - media bus frame description flags
 *
 * @V4L2_MBUS_FRAME_DESC_FL_LEN_MAX:
 *        Indicates that &struct v4l2_mbus_frame_desc_entry->length field
 *        specifies maximum data length.
 * @V4L2_MBUS_FRAME_DESC_FL_BLOB:
 *        Indicates that the format does not have line offsets, i.e.
 *        the receiver should use 1D DMA.
 */
enum v4l2_mbus_frame_desc_flags {
        V4L2_MBUS_FRAME_DESC_FL_LEN_MAX        = BIT(0),
        V4L2_MBUS_FRAME_DESC_FL_BLOB        = BIT(1),
};

/**
 * struct v4l2_mbus_frame_desc_entry - media bus frame description structure
 *
 * @flags:        bitmask flags, as defined by &enum v4l2_mbus_frame_desc_flags.
 * @stream:        stream in routing configuration
 * @pixelcode:        media bus pixel code, valid if @flags
 *                %FRAME_DESC_FL_BLOB is not set.
 * @length:        number of octets per frame, valid if @flags
 *                %V4L2_MBUS_FRAME_DESC_FL_LEN_MAX is set.
 * @bus:        Bus-specific frame descriptor parameters
 * @bus.csi2:        CSI-2-specific bus configuration
 */
struct v4l2_mbus_frame_desc_entry {
        enum v4l2_mbus_frame_desc_flags flags;
        u32 stream;
        u32 pixelcode;
        u32 length;
        union {
                struct v4l2_mbus_frame_desc_entry_csi2 csi2;
        } bus;
};

 /*
  * If this number is too small, it should be dropped altogether and the
  * API switched to a dynamic number of frame descriptor entries.
  */
#define V4L2_FRAME_DESC_ENTRY_MAX        8

/**
 * enum v4l2_mbus_frame_desc_type - media bus frame description type
 *
 * @V4L2_MBUS_FRAME_DESC_TYPE_UNDEFINED:
 *        Undefined frame desc type. Drivers should not use this, it is
 *        for backwards compatibility.
 * @V4L2_MBUS_FRAME_DESC_TYPE_PARALLEL:
 *        Parallel media bus.
 * @V4L2_MBUS_FRAME_DESC_TYPE_CSI2:
 *        CSI-2 media bus. Frame desc parameters must be set in
 *        &struct v4l2_mbus_frame_desc_entry->csi2.
 */
enum v4l2_mbus_frame_desc_type {
        V4L2_MBUS_FRAME_DESC_TYPE_UNDEFINED = 0,
        V4L2_MBUS_FRAME_DESC_TYPE_PARALLEL,
        V4L2_MBUS_FRAME_DESC_TYPE_CSI2,
};

/**
 * struct v4l2_mbus_frame_desc - media bus data frame description
 * @type: type of the bus (enum v4l2_mbus_frame_desc_type)
 * @entry: frame descriptors array
 * @num_entries: number of entries in @entry array
 */
struct v4l2_mbus_frame_desc {
        enum v4l2_mbus_frame_desc_type type;
        struct v4l2_mbus_frame_desc_entry entry[V4L2_FRAME_DESC_ENTRY_MAX];
        unsigned short num_entries;
};

/**
 * enum v4l2_subdev_pre_streamon_flags - Flags for pre_streamon subdev core op
 *
 * @V4L2_SUBDEV_PRE_STREAMON_FL_MANUAL_LP: Set the transmitter to either LP-11
 *        or LP-111 mode before call to s_stream().
 */
enum v4l2_subdev_pre_streamon_flags {
        V4L2_SUBDEV_PRE_STREAMON_FL_MANUAL_LP = BIT(0),
};

/**
 * struct v4l2_subdev_video_ops - Callbacks used when v4l device was opened
 *                                  in video mode.
 *
 * @s_routing: see s_routing in audio_ops, except this version is for video
 *        devices.
 *
 * @s_crystal_freq: sets the frequency of the crystal used to generate the
 *        clocks in Hz. An extra flags field allows device specific configuration
 *        regarding clock frequency dividers, etc. If not used, then set flags
 *        to 0. If the frequency is not supported, then -EINVAL is returned.
 *
 * @g_std: callback for VIDIOC_G_STD() ioctl handler code.
 *
 * @s_std: callback for VIDIOC_S_STD() ioctl handler code.
 *
 * @s_std_output: set v4l2_std_id for video OUTPUT devices. This is ignored by
 *        video input devices.
 *
 * @g_std_output: get current standard for video OUTPUT devices. This is ignored
 *        by video input devices.
 *
 * @querystd: callback for VIDIOC_QUERYSTD() ioctl handler code.
 *
 * @g_tvnorms: get &v4l2_std_id with all standards supported by the video
 *        CAPTURE device. This is ignored by video output devices.
 *
 * @g_tvnorms_output: get v4l2_std_id with all standards supported by the video
 *        OUTPUT device. This is ignored by video capture devices.
 *
 * @g_input_status: get input status. Same as the status field in the
 *        &struct v4l2_input
 *
 * @s_stream: start (enabled == 1) or stop (enabled == 0) streaming on the
 *        sub-device. Failure on stop will remove any resources acquired in
 *        streaming start, while the error code is still returned by the driver.
 *        The caller shall track the subdev state, and shall not start or stop an
 *        already started or stopped subdev. Also see call_s_stream wrapper in
 *        v4l2-subdev.c.
 *
 *        This callback is DEPRECATED. New drivers should instead implement
 *        &v4l2_subdev_pad_ops.enable_streams and
 *        &v4l2_subdev_pad_ops.disable_streams operations, and use
 *        v4l2_subdev_s_stream_helper for the &v4l2_subdev_video_ops.s_stream
 *        operation to support legacy users.
 *
 *        Drivers should also not call the .s_stream() subdev operation directly,
 *        but use the v4l2_subdev_enable_streams() and
 *        v4l2_subdev_disable_streams() helpers.
 *
 * @g_pixelaspect: callback to return the pixelaspect ratio.
 *
 * @s_rx_buffer: set a host allocated memory buffer for the subdev. The subdev
 *        can adjust @size to a lower value and must not write more data to the
 *        buffer starting at @data than the original value of @size.
 *
 * @pre_streamon: May be called before streaming is actually started, to help
 *        initialising the bus. Current usage is to set a CSI-2 transmitter to
 *        LP-11 or LP-111 mode before streaming. See &enum
 *        v4l2_subdev_pre_streamon_flags.
 *
 *        pre_streamon shall return error if it cannot perform the operation as
 *        indicated by the flags argument. In particular, -EACCES indicates lack
 *        of support for the operation. The caller shall call post_streamoff for
 *        each successful call of pre_streamon.
 *
 * @post_streamoff: Called after streaming is stopped, but if and only if
 *        pre_streamon was called earlier.
 */
struct v4l2_subdev_video_ops {
        int (*s_routing)(struct v4l2_subdev *sd, u32 input, u32 output, u32 config);
        int (*s_crystal_freq)(struct v4l2_subdev *sd, u32 freq, u32 flags);
        int (*g_std)(struct v4l2_subdev *sd, v4l2_std_id *norm);
        int (*s_std)(struct v4l2_subdev *sd, v4l2_std_id norm);
        int (*s_std_output)(struct v4l2_subdev *sd, v4l2_std_id std);
        int (*g_std_output)(struct v4l2_subdev *sd, v4l2_std_id *std);
        int (*querystd)(struct v4l2_subdev *sd, v4l2_std_id *std);
        int (*g_tvnorms)(struct v4l2_subdev *sd, v4l2_std_id *std);
        int (*g_tvnorms_output)(struct v4l2_subdev *sd, v4l2_std_id *std);
        int (*g_input_status)(struct v4l2_subdev *sd, u32 *status);
        int (*s_stream)(struct v4l2_subdev *sd, int enable);
        int (*g_pixelaspect)(struct v4l2_subdev *sd, struct v4l2_fract *aspect);
        int (*s_rx_buffer)(struct v4l2_subdev *sd, void *buf,
                           unsigned int *size);
        int (*pre_streamon)(struct v4l2_subdev *sd, u32 flags);
        int (*post_streamoff)(struct v4l2_subdev *sd);
};

/**
 * struct v4l2_subdev_vbi_ops - Callbacks used when v4l device was opened
 *                                  in video mode via the vbi device node.
 *
 *  @decode_vbi_line: video decoders that support sliced VBI need to implement
 *        this ioctl. Field p of the &struct v4l2_decode_vbi_line is set to the
 *        start of the VBI data that was generated by the decoder. The driver
 *        then parses the sliced VBI data and sets the other fields in the
 *        struct accordingly. The pointer p is updated to point to the start of
 *        the payload which can be copied verbatim into the data field of the
 *        &struct v4l2_sliced_vbi_data. If no valid VBI data was found, then the
 *        type field is set to 0 on return.
 *
 * @s_vbi_data: used to generate VBI signals on a video signal.
 *        &struct v4l2_sliced_vbi_data is filled with the data packets that
 *        should be output. Note that if you set the line field to 0, then that
 *        VBI signal is disabled. If no valid VBI data was found, then the type
 *        field is set to 0 on return.
 *
 * @g_vbi_data: used to obtain the sliced VBI packet from a readback register.
 *        Not all video decoders support this. If no data is available because
 *        the readback register contains invalid or erroneous data %-EIO is
 *        returned. Note that you must fill in the 'id' member and the 'field'
 *        member (to determine whether CC data from the first or second field
 *        should be obtained).
 *
 * @g_sliced_vbi_cap: callback for VIDIOC_G_SLICED_VBI_CAP() ioctl handler
 *                      code.
 *
 * @s_raw_fmt: setup the video encoder/decoder for raw VBI.
 *
 * @g_sliced_fmt: retrieve the current sliced VBI settings.
 *
 * @s_sliced_fmt: setup the sliced VBI settings.
 */
struct v4l2_subdev_vbi_ops {
        int (*decode_vbi_line)(struct v4l2_subdev *sd, struct v4l2_decode_vbi_line *vbi_line);
        int (*s_vbi_data)(struct v4l2_subdev *sd, const struct v4l2_sliced_vbi_data *vbi_data);
        int (*g_vbi_data)(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_data *vbi_data);
        int (*g_sliced_vbi_cap)(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_cap *cap);
        int (*s_raw_fmt)(struct v4l2_subdev *sd, struct v4l2_vbi_format *fmt);
        int (*g_sliced_fmt)(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_format *fmt);
        int (*s_sliced_fmt)(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_format *fmt);
};

/**
 * struct v4l2_subdev_sensor_ops - v4l2-subdev sensor operations
 * @g_skip_top_lines: number of lines at the top of the image to be skipped.
 *                      This is needed for some sensors, which always corrupt
 *                      several top lines of the output image, or which send their
 *                      metadata in them.
 * @g_skip_frames: number of frames to skip at stream start. This is needed for
 *                   buggy sensors that generate faulty frames when they are
 *                   turned on.
 */
struct v4l2_subdev_sensor_ops {
        int (*g_skip_top_lines)(struct v4l2_subdev *sd, u32 *lines);
        int (*g_skip_frames)(struct v4l2_subdev *sd, u32 *frames);
};

/**
 * enum v4l2_subdev_ir_mode- describes the type of IR supported
 *
 * @V4L2_SUBDEV_IR_MODE_PULSE_WIDTH: IR uses struct ir_raw_event records
 */
enum v4l2_subdev_ir_mode {
        V4L2_SUBDEV_IR_MODE_PULSE_WIDTH,
};

/**
 * struct v4l2_subdev_ir_parameters - Parameters for IR TX or TX
 *
 * @bytes_per_data_element: bytes per data element of data in read or
 *        write call.
 * @mode: IR mode as defined by &enum v4l2_subdev_ir_mode.
 * @enable: device is active if true
 * @interrupt_enable: IR interrupts are enabled if true
 * @shutdown: if true: set hardware to low/no power, false: normal mode
 *
 * @modulation: if true, it uses carrier, if false: baseband
 * @max_pulse_width:  maximum pulse width in ns, valid only for baseband signal
 * @carrier_freq: carrier frequency in Hz, valid only for modulated signal
 * @duty_cycle: duty cycle percentage, valid only for modulated signal
 * @invert_level: invert signal level
 *
 * @invert_carrier_sense: Send 0/space as a carrier burst. used only in TX.
 *
 * @noise_filter_min_width: min time of a valid pulse, in ns. Used only for RX.
 * @carrier_range_lower: Lower carrier range, in Hz, valid only for modulated
 *        signal. Used only for RX.
 * @carrier_range_upper: Upper carrier range, in Hz, valid only for modulated
 *        signal. Used only for RX.
 * @resolution: The receive resolution, in ns . Used only for RX.
 */
struct v4l2_subdev_ir_parameters {
        unsigned int bytes_per_data_element;
        enum v4l2_subdev_ir_mode mode;

        bool enable;
        bool interrupt_enable;
        bool shutdown;

        bool modulation;
        u32 max_pulse_width;
        unsigned int carrier_freq;
        unsigned int duty_cycle;
        bool invert_level;

        /* Tx only */
        bool invert_carrier_sense;

        /* Rx only */
        u32 noise_filter_min_width;
        unsigned int carrier_range_lower;
        unsigned int carrier_range_upper;
        u32 resolution;
};

/**
 * struct v4l2_subdev_ir_ops - operations for IR subdevices
 *
 * @rx_read: Reads received codes or pulse width data.
 *        The semantics are similar to a non-blocking read() call.
 * @rx_g_parameters: Get the current operating parameters and state of
 *        the IR receiver.
 * @rx_s_parameters: Set the current operating parameters and state of
 *        the IR receiver.  It is recommended to call
 *        [rt]x_g_parameters first to fill out the current state, and only change
 *        the fields that need to be changed.  Upon return, the actual device
 *        operating parameters and state will be returned.  Note that hardware
 *        limitations may prevent the actual settings from matching the requested
 *        settings - e.g. an actual carrier setting of 35,904 Hz when 36,000 Hz
 *        was requested.  An exception is when the shutdown parameter is true.
 *        The last used operational parameters will be returned, but the actual
 *        state of the hardware be different to minimize power consumption and
 *        processing when shutdown is true.
 *
 * @tx_write: Writes codes or pulse width data for transmission.
 *        The semantics are similar to a non-blocking write() call.
 * @tx_g_parameters: Get the current operating parameters and state of
 *        the IR transmitter.
 * @tx_s_parameters: Set the current operating parameters and state of
 *        the IR transmitter.  It is recommended to call
 *        [rt]x_g_parameters first to fill out the current state, and only change
 *        the fields that need to be changed.  Upon return, the actual device
 *        operating parameters and state will be returned.  Note that hardware
 *        limitations may prevent the actual settings from matching the requested
 *        settings - e.g. an actual carrier setting of 35,904 Hz when 36,000 Hz
 *        was requested.  An exception is when the shutdown parameter is true.
 *        The last used operational parameters will be returned, but the actual
 *        state of the hardware be different to minimize power consumption and
 *        processing when shutdown is true.
 */
struct v4l2_subdev_ir_ops {
        /* Receiver */
        int (*rx_read)(struct v4l2_subdev *sd, u8 *buf, size_t count,
                                ssize_t *num);

        int (*rx_g_parameters)(struct v4l2_subdev *sd,
                                struct v4l2_subdev_ir_parameters *params);
        int (*rx_s_parameters)(struct v4l2_subdev *sd,
                                struct v4l2_subdev_ir_parameters *params);

        /* Transmitter */
        int (*tx_write)(struct v4l2_subdev *sd, u8 *buf, size_t count,
                                ssize_t *num);

        int (*tx_g_parameters)(struct v4l2_subdev *sd,
                                struct v4l2_subdev_ir_parameters *params);
        int (*tx_s_parameters)(struct v4l2_subdev *sd,
                                struct v4l2_subdev_ir_parameters *params);
};

/**
 * struct v4l2_subdev_pad_config - Used for storing subdev pad information.
 *
 * @format: &struct v4l2_mbus_framefmt
 * @crop: &struct v4l2_rect to be used for crop
 * @compose: &struct v4l2_rect to be used for compose
 * @interval: frame interval
 */
struct v4l2_subdev_pad_config {
        struct v4l2_mbus_framefmt format;
        struct v4l2_rect crop;
        struct v4l2_rect compose;
        struct v4l2_fract interval;
};

/**
 * struct v4l2_subdev_stream_config - Used for storing stream configuration.
 *
 * @pad: pad number
 * @stream: stream number
 * @enabled: has the stream been enabled with v4l2_subdev_enable_streams()
 * @fmt: &struct v4l2_mbus_framefmt
 * @crop: &struct v4l2_rect to be used for crop
 * @compose: &struct v4l2_rect to be used for compose
 * @interval: frame interval
 *
 * This structure stores configuration for a stream.
 */
struct v4l2_subdev_stream_config {
        u32 pad;
        u32 stream;
        bool enabled;

        struct v4l2_mbus_framefmt fmt;
        struct v4l2_rect crop;
        struct v4l2_rect compose;
        struct v4l2_fract interval;
};

/**
 * struct v4l2_subdev_stream_configs - A collection of stream configs.
 *
 * @num_configs: number of entries in @config.
 * @configs: an array of &struct v4l2_subdev_stream_configs.
 */
struct v4l2_subdev_stream_configs {
        u32 num_configs;
        struct v4l2_subdev_stream_config *configs;
};

/**
 * struct v4l2_subdev_krouting - subdev routing table
 *
 * @len_routes: length of routes array, in routes
 * @num_routes: number of routes
 * @routes: &struct v4l2_subdev_route
 *
 * This structure contains the routing table for a subdev.
 */
struct v4l2_subdev_krouting {
        unsigned int len_routes;
        unsigned int num_routes;
        struct v4l2_subdev_route *routes;
};

/**
 * struct v4l2_subdev_state - Used for storing subdev state information.
 *
 * @_lock: default for 'lock'
 * @lock: mutex for the state. May be replaced by the user.
 * @sd: the sub-device which the state is related to
 * @pads: &struct v4l2_subdev_pad_config array
 * @routing: routing table for the subdev
 * @stream_configs: stream configurations (only for V4L2_SUBDEV_FL_STREAMS)
 *
 * This structure only needs to be passed to the pad op if the 'which' field
 * of the main argument is set to %V4L2_SUBDEV_FORMAT_TRY. For
 * %V4L2_SUBDEV_FORMAT_ACTIVE it is safe to pass %NULL.
 */
struct v4l2_subdev_state {
        /* lock for the struct v4l2_subdev_state fields */
        struct mutex _lock;
        struct mutex *lock;
        struct v4l2_subdev *sd;
        struct v4l2_subdev_pad_config *pads;
        struct v4l2_subdev_krouting routing;
        struct v4l2_subdev_stream_configs stream_configs;
};

/**
 * struct v4l2_subdev_pad_ops - v4l2-subdev pad level operations
 *
 * @enum_mbus_code: callback for VIDIOC_SUBDEV_ENUM_MBUS_CODE() ioctl handler
 *                    code.
 * @enum_frame_size: callback for VIDIOC_SUBDEV_ENUM_FRAME_SIZE() ioctl handler
 *                     code.
 *
 * @enum_frame_interval: callback for VIDIOC_SUBDEV_ENUM_FRAME_INTERVAL() ioctl
 *                         handler code.
 *
 * @get_fmt: callback for VIDIOC_SUBDEV_G_FMT() ioctl handler code.
 *
 * @set_fmt: callback for VIDIOC_SUBDEV_S_FMT() ioctl handler code.
 *
 * @get_selection: callback for VIDIOC_SUBDEV_G_SELECTION() ioctl handler code.
 *
 * @set_selection: callback for VIDIOC_SUBDEV_S_SELECTION() ioctl handler code.
 *
 * @get_frame_interval: callback for VIDIOC_SUBDEV_G_FRAME_INTERVAL()
 *                        ioctl handler code.
 *
 * @set_frame_interval: callback for VIDIOC_SUBDEV_S_FRAME_INTERVAL()
 *                        ioctl handler code.
 *
 * @get_edid: callback for VIDIOC_SUBDEV_G_EDID() ioctl handler code.
 *
 * @set_edid: callback for VIDIOC_SUBDEV_S_EDID() ioctl handler code.
 *
 * @s_dv_timings: Set custom dv timings in the sub device. This is used
 *        when sub device is capable of setting detailed timing information
 *        in the hardware to generate/detect the video signal.
 *
 * @g_dv_timings: Get custom dv timings in the sub device.
 *
 * @query_dv_timings: callback for VIDIOC_QUERY_DV_TIMINGS() ioctl handler code.
 *
 * @dv_timings_cap: callback for VIDIOC_SUBDEV_DV_TIMINGS_CAP() ioctl handler
 *                    code.
 *
 * @enum_dv_timings: callback for VIDIOC_SUBDEV_ENUM_DV_TIMINGS() ioctl handler
 *                     code.
 *
 * @link_validate: used by the media controller code to check if the links
 *                   that belongs to a pipeline can be used for stream.
 *
 * @get_frame_desc: get the current low level media bus frame parameters.
 *
 * @set_frame_desc: set the low level media bus frame parameters, @fd array
 *                  may be adjusted by the subdev driver to device capabilities.
 *
 * @get_mbus_config: get the media bus configuration of a remote sub-device.
 *                     The media bus configuration is usually retrieved from the
 *                     firmware interface at sub-device probe time, immediately
 *                     applied to the hardware and eventually adjusted by the
 *                     driver. Remote sub-devices (usually video receivers) shall
 *                     use this operation to query the transmitting end bus
 *                     configuration in order to adjust their own one accordingly.
 *                     Callers should make sure they get the most up-to-date as
 *                     possible configuration from the remote end, likely calling
 *                     this operation as close as possible to stream on time. The
 *                     operation shall fail if the pad index it has been called on
 *                     is not valid or in case of unrecoverable failures. The
 *                     config argument has been memset to 0 just before calling
 *                     the op.
 *
 * @set_routing: Enable or disable data connection routes described in the
 *                 subdevice routing table. Subdevs that implement this operation
 *                 must set the V4L2_SUBDEV_FL_STREAMS flag.
 *
 * @enable_streams: Enable the streams defined in streams_mask on the given
 *        source pad. Subdevs that implement this operation must use the active
 *        state management provided by the subdev core (enabled through a call to
 *        v4l2_subdev_init_finalize() at initialization time). Do not call
 *        directly, use v4l2_subdev_enable_streams() instead.
 *
 *        Drivers that support only a single stream without setting the
 *        V4L2_SUBDEV_CAP_STREAMS sub-device capability flag can ignore the mask
 *        argument.
 *
 * @disable_streams: Disable the streams defined in streams_mask on the given
 *        source pad. Subdevs that implement this operation must use the active
 *        state management provided by the subdev core (enabled through a call to
 *        v4l2_subdev_init_finalize() at initialization time). Do not call
 *        directly, use v4l2_subdev_disable_streams() instead.
 *
 *        Drivers that support only a single stream without setting the
 *        V4L2_SUBDEV_CAP_STREAMS sub-device capability flag can ignore the mask
 *        argument.
 */
struct v4l2_subdev_pad_ops {
        int (*enum_mbus_code)(struct v4l2_subdev *sd,
                              struct v4l2_subdev_state *state,
                              struct v4l2_subdev_mbus_code_enum *code);
        int (*enum_frame_size)(struct v4l2_subdev *sd,
                               struct v4l2_subdev_state *state,
                               struct v4l2_subdev_frame_size_enum *fse);
        int (*enum_frame_interval)(struct v4l2_subdev *sd,
                                   struct v4l2_subdev_state *state,
                                   struct v4l2_subdev_frame_interval_enum *fie);
        int (*get_fmt)(struct v4l2_subdev *sd,
                       struct v4l2_subdev_state *state,
                       struct v4l2_subdev_format *format);
        int (*set_fmt)(struct v4l2_subdev *sd,
                       struct v4l2_subdev_state *state,
                       struct v4l2_subdev_format *format);
        int (*get_selection)(struct v4l2_subdev *sd,
                             struct v4l2_subdev_state *state,
                             struct v4l2_subdev_selection *sel);
        int (*set_selection)(struct v4l2_subdev *sd,
                             struct v4l2_subdev_state *state,
                             struct v4l2_subdev_selection *sel);
        int (*get_frame_interval)(struct v4l2_subdev *sd,
                                  struct v4l2_subdev_state *state,
                                  struct v4l2_subdev_frame_interval *interval);
        int (*set_frame_interval)(struct v4l2_subdev *sd,
                                  struct v4l2_subdev_state *state,
                                  struct v4l2_subdev_frame_interval *interval);
        int (*get_edid)(struct v4l2_subdev *sd, struct v4l2_edid *edid);
        int (*set_edid)(struct v4l2_subdev *sd, struct v4l2_edid *edid);
        int (*s_dv_timings)(struct v4l2_subdev *sd, unsigned int pad,
                            struct v4l2_dv_timings *timings);
        int (*g_dv_timings)(struct v4l2_subdev *sd, unsigned int pad,
                            struct v4l2_dv_timings *timings);
        int (*query_dv_timings)(struct v4l2_subdev *sd, unsigned int pad,
                                struct v4l2_dv_timings *timings);
        int (*dv_timings_cap)(struct v4l2_subdev *sd,
                              struct v4l2_dv_timings_cap *cap);
        int (*enum_dv_timings)(struct v4l2_subdev *sd,
                               struct v4l2_enum_dv_timings *timings);
#ifdef CONFIG_MEDIA_CONTROLLER
        int (*link_validate)(struct v4l2_subdev *sd, struct media_link *link,
                             struct v4l2_subdev_format *source_fmt,
                             struct v4l2_subdev_format *sink_fmt);
#endif /* CONFIG_MEDIA_CONTROLLER */
        int (*get_frame_desc)(struct v4l2_subdev *sd, unsigned int pad,
                              struct v4l2_mbus_frame_desc *fd);
        int (*set_frame_desc)(struct v4l2_subdev *sd, unsigned int pad,
                              struct v4l2_mbus_frame_desc *fd);
        int (*get_mbus_config)(struct v4l2_subdev *sd, unsigned int pad,
                               struct v4l2_mbus_config *config);
        int (*set_routing)(struct v4l2_subdev *sd,
                           struct v4l2_subdev_state *state,
                           enum v4l2_subdev_format_whence which,
                           struct v4l2_subdev_krouting *route);
        int (*enable_streams)(struct v4l2_subdev *sd,
                              struct v4l2_subdev_state *state, u32 pad,
                              u64 streams_mask);
        int (*disable_streams)(struct v4l2_subdev *sd,
                               struct v4l2_subdev_state *state, u32 pad,
                               u64 streams_mask);
};

/**
 * struct v4l2_subdev_ops - Subdev operations
 *
 * @core: pointer to &struct v4l2_subdev_core_ops. Can be %NULL
 * @tuner: pointer to &struct v4l2_subdev_tuner_ops. Can be %NULL
 * @audio: pointer to &struct v4l2_subdev_audio_ops. Can be %NULL
 * @video: pointer to &struct v4l2_subdev_video_ops. Can be %NULL
 * @vbi: pointer to &struct v4l2_subdev_vbi_ops. Can be %NULL
 * @ir: pointer to &struct v4l2_subdev_ir_ops. Can be %NULL
 * @sensor: pointer to &struct v4l2_subdev_sensor_ops. Can be %NULL
 * @pad: pointer to &struct v4l2_subdev_pad_ops. Can be %NULL
 */
struct v4l2_subdev_ops {
        const struct v4l2_subdev_core_ops        *core;
        const struct v4l2_subdev_tuner_ops        *tuner;
        const struct v4l2_subdev_audio_ops        *audio;
        const struct v4l2_subdev_video_ops        *video;
        const struct v4l2_subdev_vbi_ops        *vbi;
        const struct v4l2_subdev_ir_ops                *ir;
        const struct v4l2_subdev_sensor_ops        *sensor;
        const struct v4l2_subdev_pad_ops        *pad;
};

/**
 * struct v4l2_subdev_internal_ops - V4L2 subdev internal ops
 *
 * @init_state: initialize the subdev state to default values
 *
 * @registered: called when this subdev is registered. When called the v4l2_dev
 *        field is set to the correct v4l2_device.
 *
 * @unregistered: called when this subdev is unregistered. When called the
 *        v4l2_dev field is still set to the correct v4l2_device.
 *
 * @open: called when the subdev device node is opened by an application.
 *
 * @close: called when the subdev device node is closed. Please note that
 *        it is possible for @close to be called after @unregistered!
 *
 * @release: called when the last user of the subdev device is gone. This
 *        happens after the @unregistered callback and when the last open
 *        filehandle to the v4l-subdevX device node was closed. If no device
 *        node was created for this sub-device, then the @release callback
 *        is called right after the @unregistered callback.
 *        The @release callback is typically used to free the memory containing
 *        the v4l2_subdev structure. It is almost certainly required for any
 *        sub-device that sets the V4L2_SUBDEV_FL_HAS_DEVNODE flag.
 *
 * .. note::
 *        Never call this from drivers, only the v4l2 framework can call
 *        these ops.
 */
struct v4l2_subdev_internal_ops {
        int (*init_state)(struct v4l2_subdev *sd,
                          struct v4l2_subdev_state *state);
        int (*registered)(struct v4l2_subdev *sd);
        void (*unregistered)(struct v4l2_subdev *sd);
        int (*open)(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh);
        int (*close)(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh);
        void (*release)(struct v4l2_subdev *sd);
};

/* Set this flag if this subdev is a i2c device. */
#define V4L2_SUBDEV_FL_IS_I2C                        (1U << 0)
/* Set this flag if this subdev is a spi device. */
#define V4L2_SUBDEV_FL_IS_SPI                        (1U << 1)
/* Set this flag if this subdev needs a device node. */
#define V4L2_SUBDEV_FL_HAS_DEVNODE                (1U << 2)
/*
 * Set this flag if this subdev generates events.
 * Note controls can send events, thus drivers exposing controls
 * should set this flag.
 */
#define V4L2_SUBDEV_FL_HAS_EVENTS                (1U << 3)
/*
 * Set this flag if this subdev supports multiplexed streams. This means
 * that the driver supports routing and handles the stream parameter in its
 * v4l2_subdev_pad_ops handlers. More specifically, this means:
 *
 * - Centrally managed subdev active state is enabled
 * - Legacy pad config is _not_ supported (state->pads is NULL)
 * - Routing ioctls are available
 * - Multiple streams per pad are supported
 */
#define V4L2_SUBDEV_FL_STREAMS                        (1U << 4)

struct regulator_bulk_data;

/**
 * struct v4l2_subdev_platform_data - regulators config struct
 *
 * @regulators: Optional regulators used to power on/off the subdevice
 * @num_regulators: Number of regululators
 * @host_priv: Per-subdevice data, specific for a certain video host device
 */
struct v4l2_subdev_platform_data {
        struct regulator_bulk_data *regulators;
        int num_regulators;

        void *host_priv;
};

/**
 * struct v4l2_subdev - describes a V4L2 sub-device
 *
 * @entity: pointer to &struct media_entity
 * @list: List of sub-devices
 * @owner: The owner is the same as the driver's &struct device owner.
 * @owner_v4l2_dev: true if the &sd->owner matches the owner of @v4l2_dev->dev
 *        owner. Initialized by v4l2_device_register_subdev().
 * @flags: subdev flags. Can be:
 *   %V4L2_SUBDEV_FL_IS_I2C - Set this flag if this subdev is a i2c device;
 *   %V4L2_SUBDEV_FL_IS_SPI - Set this flag if this subdev is a spi device;
 *   %V4L2_SUBDEV_FL_HAS_DEVNODE - Set this flag if this subdev needs a
 *   device node;
 *   %V4L2_SUBDEV_FL_HAS_EVENTS -  Set this flag if this subdev generates
 *   events.
 *
 * @v4l2_dev: pointer to struct &v4l2_device
 * @ops: pointer to struct &v4l2_subdev_ops
 * @internal_ops: pointer to struct &v4l2_subdev_internal_ops.
 *        Never call these internal ops from within a driver!
 * @ctrl_handler: The control handler of this subdev. May be NULL.
 * @name: Name of the sub-device. Please notice that the name must be unique.
 * @grp_id: can be used to group similar subdevs. Value is driver-specific
 * @dev_priv: pointer to private data
 * @host_priv: pointer to private data used by the device where the subdev
 *        is attached.
 * @devnode: subdev device node
 * @dev: pointer to the physical device, if any
 * @fwnode: The fwnode_handle of the subdev, usually the same as
 *            either dev->of_node->fwnode or dev->fwnode (whichever is non-NULL).
 * @async_list: Links this subdev to a global subdev_list or
 *                @notifier->done_list list.
 * @async_subdev_endpoint_list: List entry in async_subdev_endpoint_entry of
 *                                &struct v4l2_async_subdev_endpoint.
 * @subdev_notifier: A sub-device notifier implicitly registered for the sub-
 *                     device using v4l2_async_register_subdev_sensor().
 * @asc_list: Async connection list, of &struct
 *              v4l2_async_connection.subdev_entry.
 * @pdata: common part of subdevice platform data
 * @state_lock: A pointer to a lock used for all the subdev's states, set by the
 *                driver. This is        optional. If NULL, each state instance will get
 *                a lock of its own.
 * @privacy_led: Optional pointer to a LED classdev for the privacy LED for sensors.
 * @active_state: Active state for the subdev (NULL for subdevs tracking the
 *                  state internally). Initialized by calling
 *                  v4l2_subdev_init_finalize().
 * @enabled_pads: Bitmask of enabled pads used by v4l2_subdev_enable_streams()
 *                  and v4l2_subdev_disable_streams() helper functions for
 *                  fallback cases.
 * @s_stream_enabled: Tracks whether streaming has been enabled with s_stream.
 *                    This is only for call_s_stream() internal use.
 *
 * Each instance of a subdev driver should create this struct, either
 * stand-alone or embedded in a larger struct.
 *
 * This structure should be initialized by v4l2_subdev_init() or one of
 * its variants: v4l2_spi_subdev_init(), v4l2_i2c_subdev_init().
 */
struct v4l2_subdev {
#if defined(CONFIG_MEDIA_CONTROLLER)
        struct media_entity entity;
#endif
        struct list_head list;
        struct module *owner;
        bool owner_v4l2_dev;
        u32 flags;
        struct v4l2_device *v4l2_dev;
        const struct v4l2_subdev_ops *ops;
        const struct v4l2_subdev_internal_ops *internal_ops;
        struct v4l2_ctrl_handler *ctrl_handler;
        char name[52];
        u32 grp_id;
        void *dev_priv;
        void *host_priv;
        struct video_device *devnode;
        struct device *dev;
        struct fwnode_handle *fwnode;
        struct list_head async_list;
        struct list_head async_subdev_endpoint_list;
        struct v4l2_async_notifier *subdev_notifier;
        struct list_head asc_list;
        struct v4l2_subdev_platform_data *pdata;
        struct mutex *state_lock;

        /*
         * The fields below are private, and should only be accessed via
         * appropriate functions.
         */

        struct led_classdev *privacy_led;

        /*
         * TODO: active_state should most likely be changed from a pointer to an
         * embedded field. For the time being it's kept as a pointer to more
         * easily catch uses of active_state in the cases where the driver
         * doesn't support it.
         */
        struct v4l2_subdev_state *active_state;
        u64 enabled_pads;
        bool s_stream_enabled;
};


/**
 * media_entity_to_v4l2_subdev - Returns a &struct v4l2_subdev from
 *    the &struct media_entity embedded in it.
 *
 * @ent: pointer to &struct media_entity.
 */
#define media_entity_to_v4l2_subdev(ent)                                \
({                                                                        \
        typeof(ent) __me_sd_ent = (ent);                                \
                                                                        \
        __me_sd_ent ?                                                        \
                container_of(__me_sd_ent, struct v4l2_subdev, entity) :        \
                NULL;                                                        \
})

/**
 * vdev_to_v4l2_subdev - Returns a &struct v4l2_subdev from
 *        the &struct video_device embedded on it.
 *
 * @vdev: pointer to &struct video_device
 */
#define vdev_to_v4l2_subdev(vdev) \
        ((struct v4l2_subdev *)video_get_drvdata(vdev))

/**
 * struct v4l2_subdev_fh - Used for storing subdev information per file handle
 *
 * @vfh: pointer to &struct v4l2_fh
 * @state: pointer to &struct v4l2_subdev_state
 * @owner: module pointer to the owner of this file handle
 * @client_caps: bitmask of ``V4L2_SUBDEV_CLIENT_CAP_*``
 */
struct v4l2_subdev_fh {
        struct v4l2_fh vfh;
        struct module *owner;
#if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)
        struct v4l2_subdev_state *state;
        u64 client_caps;
#endif
};

/**
 * to_v4l2_subdev_fh - Returns a &struct v4l2_subdev_fh from
 *        the &struct v4l2_fh embedded on it.
 *
 * @fh: pointer to &struct v4l2_fh
 */
#define to_v4l2_subdev_fh(fh)        \
        container_of(fh, struct v4l2_subdev_fh, vfh)

extern const struct v4l2_file_operations v4l2_subdev_fops;

/**
 * v4l2_set_subdevdata - Sets V4L2 dev private device data
 *
 * @sd: pointer to &struct v4l2_subdev
 * @p: pointer to the private device data to be stored.
 */
static inline void v4l2_set_subdevdata(struct v4l2_subdev *sd, void *p)
{
        sd->dev_priv = p;
}

/**
 * v4l2_get_subdevdata - Gets V4L2 dev private device data
 *
 * @sd: pointer to &struct v4l2_subdev
 *
 * Returns the pointer to the private device data to be stored.
 */
static inline void *v4l2_get_subdevdata(const struct v4l2_subdev *sd)
{
        return sd->dev_priv;
}

/**
 * v4l2_set_subdev_hostdata - Sets V4L2 dev private host data
 *
 * @sd: pointer to &struct v4l2_subdev
 * @p: pointer to the private data to be stored.
 */
static inline void v4l2_set_subdev_hostdata(struct v4l2_subdev *sd, void *p)
{
        sd->host_priv = p;
}

/**
 * v4l2_get_subdev_hostdata - Gets V4L2 dev private data
 *
 * @sd: pointer to &struct v4l2_subdev
 *
 * Returns the pointer to the private host data to be stored.
 */
static inline void *v4l2_get_subdev_hostdata(const struct v4l2_subdev *sd)
{
        return sd->host_priv;
}

#ifdef CONFIG_MEDIA_CONTROLLER

/**
 * v4l2_subdev_get_fwnode_pad_1_to_1 - Get pad number from a subdev fwnode
 *                                     endpoint, assuming 1:1 port:pad
 *
 * @entity: Pointer to the subdev entity
 * @endpoint: Pointer to a parsed fwnode endpoint
 *
 * This function can be used as the .get_fwnode_pad operation for
 * subdevices that map port numbers and pad indexes 1:1. If the endpoint
 * is owned by the subdevice, the function returns the endpoint port
 * number.
 *
 * Returns the endpoint port number on success or a negative error code.
 */
int v4l2_subdev_get_fwnode_pad_1_to_1(struct media_entity *entity,
                                      struct fwnode_endpoint *endpoint);

/**
 * v4l2_subdev_link_validate_default - validates a media link
 *
 * @sd: pointer to &struct v4l2_subdev
 * @link: pointer to &struct media_link
 * @source_fmt: pointer to &struct v4l2_subdev_format
 * @sink_fmt: pointer to &struct v4l2_subdev_format
 *
 * This function ensures that width, height and the media bus pixel
 * code are equal on both source and sink of the link.
 */
int v4l2_subdev_link_validate_default(struct v4l2_subdev *sd,
                                      struct media_link *link,
                                      struct v4l2_subdev_format *source_fmt,
                                      struct v4l2_subdev_format *sink_fmt);

/**
 * v4l2_subdev_link_validate - validates a media link
 *
 * @link: pointer to &struct media_link
 *
 * This function calls the subdev's link_validate ops to validate
 * if a media link is valid for streaming. It also internally
 * calls v4l2_subdev_link_validate_default() to ensure that
 * width, height and the media bus pixel code are equal on both
 * source and sink of the link.
 *
 * The function can be used as a drop-in &media_entity_ops.link_validate
 * implementation for v4l2_subdev instances. It supports all links between
 * subdevs, as well as links between subdevs and video devices, provided that
 * the video devices also implement their &media_entity_ops.link_validate
 * operation.
 */
int v4l2_subdev_link_validate(struct media_link *link);

/**
 * v4l2_subdev_has_pad_interdep - MC has_pad_interdep implementation for subdevs
 *
 * @entity: pointer to &struct media_entity
 * @pad0: pad number for the first pad
 * @pad1: pad number for the second pad
 *
 * This function is an implementation of the
 * media_entity_operations.has_pad_interdep operation for subdevs that
 * implement the multiplexed streams API (as indicated by the
 * V4L2_SUBDEV_FL_STREAMS subdev flag).
 *
 * It considers two pads interdependent if there is an active route between pad0
 * and pad1.
 */
bool v4l2_subdev_has_pad_interdep(struct media_entity *entity,
                                  unsigned int pad0, unsigned int pad1);

/**
 * __v4l2_subdev_state_alloc - allocate v4l2_subdev_state
 *
 * @sd: pointer to &struct v4l2_subdev for which the state is being allocated.
 * @lock_name: name of the state lock
 * @key: lock_class_key for the lock
 *
 * Must call __v4l2_subdev_state_free() when state is no longer needed.
 *
 * Not to be called directly by the drivers.
 */
struct v4l2_subdev_state *__v4l2_subdev_state_alloc(struct v4l2_subdev *sd,
                                                    const char *lock_name,
                                                    struct lock_class_key *key);

/**
 * __v4l2_subdev_state_free - free a v4l2_subdev_state
 *
 * @state: v4l2_subdev_state to be freed.
 *
 * Not to be called directly by the drivers.
 */
void __v4l2_subdev_state_free(struct v4l2_subdev_state *state);

/**
 * v4l2_subdev_init_finalize() - Finalizes the initialization of the subdevice
 * @sd: The subdev
 *
 * This function finalizes the initialization of the subdev, including
 * allocation of the active state for the subdev.
 *
 * This function must be called by the subdev drivers that use the centralized
 * active state, after the subdev struct has been initialized and
 * media_entity_pads_init() has been called, but before registering the
 * subdev.
 *
 * The user must call v4l2_subdev_cleanup() when the subdev is being removed.
 */
#define v4l2_subdev_init_finalize(sd)                                          \
        ({                                                                     \
                static struct lock_class_key __key;                            \
                const char *name = KBUILD_BASENAME                             \
                        ":" __stringify(__LINE__) ":sd->active_state->lock";   \
                __v4l2_subdev_init_finalize(sd, name, &__key);                 \
        })

int __v4l2_subdev_init_finalize(struct v4l2_subdev *sd, const char *name,
                                struct lock_class_key *key);

/**
 * v4l2_subdev_cleanup() - Releases the resources allocated by the subdevice
 * @sd: The subdevice
 *
 * Clean up a V4L2 async sub-device. Must be called for a sub-device as part of
 * its release if resources have been associated with it using
 * v4l2_async_subdev_endpoint_add() or v4l2_subdev_init_finalize().
 */
void v4l2_subdev_cleanup(struct v4l2_subdev *sd);

/*
 * A macro to generate the macro or function name for sub-devices state access
 * wrapper macros below.
 */
#define __v4l2_subdev_state_gen_call(NAME, _1, ARG, ...)        \
        __v4l2_subdev_state_get_ ## NAME ## ARG

/*
 * A macro to constify the return value of the state accessors when the state
 * parameter is const.
 */
#define __v4l2_subdev_state_constify_ret(state, value)                                \
        _Generic(state,                                                                \
                const struct v4l2_subdev_state *: (const typeof(*(value)) *)(value), \
                struct v4l2_subdev_state *: (value)                                \
        )

/**
 * v4l2_subdev_state_get_format() - Get pointer to a stream format
 * @state: subdevice state
 * @pad: pad id
 * @...: stream id (optional argument)
 *
 * This returns a pointer to &struct v4l2_mbus_framefmt for the given pad +
 * stream in the subdev state.
 *
 * For stream-unaware drivers the format for the corresponding pad is returned.
 * If the pad does not exist, NULL is returned.
 */
/*
 * Wrap v4l2_subdev_state_get_format(), allowing the function to be called with
 * two or three arguments. The purpose of the __v4l2_subdev_state_gen_call()
 * macro is to come up with the name of the function or macro to call, using
 * the last two arguments (_stream and _pad). The selected function or macro is
 * then called using the arguments specified by the caller. The
 * __v4l2_subdev_state_constify_ret() macro constifies the returned pointer
 * when the state is const, allowing the state accessors to guarantee
 * const-correctness in all cases.
 *
 * A similar arrangement is used for v4l2_subdev_state_crop(),
 * v4l2_subdev_state_compose() and v4l2_subdev_state_get_interval() below.
 */
#define v4l2_subdev_state_get_format(state, pad, ...)                                \
        __v4l2_subdev_state_constify_ret(state,                                        \
                __v4l2_subdev_state_gen_call(format, ##__VA_ARGS__, , _pad)        \
                        ((struct v4l2_subdev_state *)state, pad, ##__VA_ARGS__))
#define __v4l2_subdev_state_get_format_pad(state, pad)        \
        __v4l2_subdev_state_get_format(state, pad, 0)
struct v4l2_mbus_framefmt *
__v4l2_subdev_state_get_format(struct v4l2_subdev_state *state,
                               unsigned int pad, u32 stream);

/**
 * v4l2_subdev_state_get_crop() - Get pointer to a stream crop rectangle
 * @state: subdevice state
 * @pad: pad id
 * @...: stream id (optional argument)
 *
 * This returns a pointer to crop rectangle for the given pad + stream in the
 * subdev state.
 *
 * For stream-unaware drivers the crop rectangle for the corresponding pad is
 * returned. If the pad does not exist, NULL is returned.
 */
#define v4l2_subdev_state_get_crop(state, pad, ...)                                \
        __v4l2_subdev_state_constify_ret(state,                                        \
                __v4l2_subdev_state_gen_call(crop, ##__VA_ARGS__, , _pad)        \
                        ((struct v4l2_subdev_state *)state, pad, ##__VA_ARGS__))
#define __v4l2_subdev_state_get_crop_pad(state, pad)        \
        __v4l2_subdev_state_get_crop(state, pad, 0)
struct v4l2_rect *
__v4l2_subdev_state_get_crop(struct v4l2_subdev_state *state, unsigned int pad,
                             u32 stream);

/**
 * v4l2_subdev_state_get_compose() - Get pointer to a stream compose rectangle
 * @state: subdevice state
 * @pad: pad id
 * @...: stream id (optional argument)
 *
 * This returns a pointer to compose rectangle for the given pad + stream in the
 * subdev state.
 *
 * For stream-unaware drivers the compose rectangle for the corresponding pad is
 * returned. If the pad does not exist, NULL is returned.
 */
#define v4l2_subdev_state_get_compose(state, pad, ...)                                \
        __v4l2_subdev_state_constify_ret(state,                                        \
                __v4l2_subdev_state_gen_call(compose, ##__VA_ARGS__, , _pad)        \
                        ((struct v4l2_subdev_state *)state, pad, ##__VA_ARGS__))
#define __v4l2_subdev_state_get_compose_pad(state, pad)        \
        __v4l2_subdev_state_get_compose(state, pad, 0)
struct v4l2_rect *
__v4l2_subdev_state_get_compose(struct v4l2_subdev_state *state,
                                unsigned int pad, u32 stream);

/**
 * v4l2_subdev_state_get_interval() - Get pointer to a stream frame interval
 * @state: subdevice state
 * @pad: pad id
 * @...: stream id (optional argument)
 *
 * This returns a pointer to the frame interval for the given pad + stream in
 * the subdev state.
 *
 * For stream-unaware drivers the frame interval for the corresponding pad is
 * returned. If the pad does not exist, NULL is returned.
 */
#define v4l2_subdev_state_get_interval(state, pad, ...)                                \
        __v4l2_subdev_state_constify_ret(state,                                        \
                __v4l2_subdev_state_gen_call(interval, ##__VA_ARGS__, , _pad)        \
                        ((struct v4l2_subdev_state *)state, pad, ##__VA_ARGS__))
#define __v4l2_subdev_state_get_interval_pad(state, pad)        \
        __v4l2_subdev_state_get_interval(state, pad, 0)
struct v4l2_fract *
__v4l2_subdev_state_get_interval(struct v4l2_subdev_state *state,
                                 unsigned int pad, u32 stream);

#if defined(CONFIG_VIDEO_V4L2_SUBDEV_API)

/**
 * v4l2_subdev_get_fmt() - Fill format based on state
 * @sd: subdevice
 * @state: subdevice state
 * @format: pointer to &struct v4l2_subdev_format
 *
 * Fill @format->format field based on the information in the @format struct.
 *
 * This function can be used by the subdev drivers which support active state to
 * implement v4l2_subdev_pad_ops.get_fmt if the subdev driver does not need to
 * do anything special in their get_fmt op.
 *
 * Returns 0 on success, error value otherwise.
 */
int v4l2_subdev_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_state *state,
                        struct v4l2_subdev_format *format);

/**
 * v4l2_subdev_get_frame_interval() - Fill frame interval based on state
 * @sd: subdevice
 * @state: subdevice state
 * @fi: pointer to &struct v4l2_subdev_frame_interval
 *
 * Fill @fi->interval field based on the information in the @fi struct.
 *
 * This function can be used by the subdev drivers which support active state to
 * implement v4l2_subdev_pad_ops.get_frame_interval if the subdev driver does
 * not need to do anything special in their get_frame_interval op.
 *
 * Returns 0 on success, error value otherwise.
 */
int v4l2_subdev_get_frame_interval(struct v4l2_subdev *sd,
                                   struct v4l2_subdev_state *state,
                                   struct v4l2_subdev_frame_interval *fi);

/**
 * v4l2_subdev_set_routing() - Set given routing to subdev state
 * @sd: The subdevice
 * @state: The subdevice state
 * @routing: Routing that will be copied to subdev state
 *
 * This will release old routing table (if any) from the state, allocate
 * enough space for the given routing, and copy the routing.
 *
 * This can be used from the subdev driver's set_routing op, after validating
 * the routing.
 */
int v4l2_subdev_set_routing(struct v4l2_subdev *sd,
                            struct v4l2_subdev_state *state,
                            const struct v4l2_subdev_krouting *routing);

struct v4l2_subdev_route *
__v4l2_subdev_next_active_route(const struct v4l2_subdev_krouting *routing,
                                struct v4l2_subdev_route *route);

/**
 * for_each_active_route - iterate on all active routes of a routing table
 * @routing: The routing table
 * @route: The route iterator
 */
#define for_each_active_route(routing, route) \
        for ((route) = NULL;                  \
             ((route) = __v4l2_subdev_next_active_route((routing), (route)));)

/**
 * v4l2_subdev_set_routing_with_fmt() - Set given routing and format to subdev
 *                                        state
 * @sd: The subdevice
 * @state: The subdevice state
 * @routing: Routing that will be copied to subdev state
 * @fmt: Format used to initialize all the streams
 *
 * This is the same as v4l2_subdev_set_routing, but additionally initializes
 * all the streams using the given format.
 */
int v4l2_subdev_set_routing_with_fmt(struct v4l2_subdev *sd,
                                     struct v4l2_subdev_state *state,
                                     const struct v4l2_subdev_krouting *routing,
                                     const struct v4l2_mbus_framefmt *fmt);

/**
 * v4l2_subdev_routing_find_opposite_end() - Find the opposite stream
 * @routing: routing used to find the opposite side
 * @pad: pad id
 * @stream: stream id
 * @other_pad: pointer used to return the opposite pad
 * @other_stream: pointer used to return the opposite stream
 *
 * This function uses the routing table to find the pad + stream which is
 * opposite the given pad + stream.
 *
 * @other_pad and/or @other_stream can be NULL if the caller does not need the
 * value.
 *
 * Returns 0 on success, or -EINVAL if no matching route is found.
 */
int v4l2_subdev_routing_find_opposite_end(const struct v4l2_subdev_krouting *routing,
                                          u32 pad, u32 stream, u32 *other_pad,
                                          u32 *other_stream);

/**
 * v4l2_subdev_state_get_opposite_stream_format() - Get pointer to opposite
 *                                                  stream format
 * @state: subdevice state
 * @pad: pad id
 * @stream: stream id
 *
 * This returns a pointer to &struct v4l2_mbus_framefmt for the pad + stream
 * that is opposite the given pad + stream in the subdev state.
 *
 * If the state does not contain the given pad + stream, NULL is returned.
 */
struct v4l2_mbus_framefmt *
v4l2_subdev_state_get_opposite_stream_format(struct v4l2_subdev_state *state,
                                             u32 pad, u32 stream);

/**
 * v4l2_subdev_state_xlate_streams() - Translate streams from one pad to another
 *
 * @state: Subdevice state
 * @pad0: The first pad
 * @pad1: The second pad
 * @streams: Streams bitmask on the first pad
 *
 * Streams on sink pads of a subdev are routed to source pads as expressed in
 * the subdev state routing table. Stream numbers don't necessarily match on
 * the sink and source side of a route. This function translates stream numbers
 * on @pad0, expressed as a bitmask in @streams, to the corresponding streams
 * on @pad1 using the routing table from the @state. It returns the stream mask
 * on @pad1, and updates @streams with the streams that have been found in the
 * routing table.
 *
 * @pad0 and @pad1 must be a sink and a source, in any order.
 *
 * Return: The bitmask of streams of @pad1 that are routed to @streams on @pad0.
 */
u64 v4l2_subdev_state_xlate_streams(const struct v4l2_subdev_state *state,
                                    u32 pad0, u32 pad1, u64 *streams);

/**
 * enum v4l2_subdev_routing_restriction - Subdevice internal routing restrictions
 *
 * @V4L2_SUBDEV_ROUTING_NO_1_TO_N:
 *        an input stream shall not be routed to multiple output streams (stream
 *        duplication)
 * @V4L2_SUBDEV_ROUTING_NO_N_TO_1:
 *        multiple input streams shall not be routed to the same output stream
 *        (stream merging)
 * @V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX:
 *        all streams from a sink pad must be routed to a single source pad
 * @V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX:
 *        all streams on a source pad must originate from a single sink pad
 * @V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING:
 *        source pads shall not contain multiplexed streams
 * @V4L2_SUBDEV_ROUTING_NO_SINK_MULTIPLEXING:
 *        sink pads shall not contain multiplexed streams
 * @V4L2_SUBDEV_ROUTING_ONLY_1_TO_1:
 *        only non-overlapping 1-to-1 stream routing is allowed (a combination of
 *        @V4L2_SUBDEV_ROUTING_NO_1_TO_N and @V4L2_SUBDEV_ROUTING_NO_N_TO_1)
 * @V4L2_SUBDEV_ROUTING_NO_STREAM_MIX:
 *        all streams from a sink pad must be routed to a single source pad, and
 *        that source pad shall not get routes from any other sink pad
 *        (a combination of @V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX and
 *        @V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX)
 * @V4L2_SUBDEV_ROUTING_NO_MULTIPLEXING:
 *        no multiplexed streams allowed on either source or sink sides.
 */
enum v4l2_subdev_routing_restriction {
        V4L2_SUBDEV_ROUTING_NO_1_TO_N = BIT(0),
        V4L2_SUBDEV_ROUTING_NO_N_TO_1 = BIT(1),
        V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX = BIT(2),
        V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX = BIT(3),
        V4L2_SUBDEV_ROUTING_NO_SINK_MULTIPLEXING = BIT(4),
        V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING = BIT(5),
        V4L2_SUBDEV_ROUTING_ONLY_1_TO_1 =
                V4L2_SUBDEV_ROUTING_NO_1_TO_N |
                V4L2_SUBDEV_ROUTING_NO_N_TO_1,
        V4L2_SUBDEV_ROUTING_NO_STREAM_MIX =
                V4L2_SUBDEV_ROUTING_NO_SINK_STREAM_MIX |
                V4L2_SUBDEV_ROUTING_NO_SOURCE_STREAM_MIX,
        V4L2_SUBDEV_ROUTING_NO_MULTIPLEXING =
                V4L2_SUBDEV_ROUTING_NO_SINK_MULTIPLEXING |
                V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING,
};

/**
 * v4l2_subdev_routing_validate() - Verify that routes comply with driver
 *                                    constraints
 * @sd: The subdevice
 * @routing: Routing to verify
 * @disallow: Restrictions on routes
 *
 * This verifies that the given routing complies with the @disallow constraints.
 *
 * Returns 0 on success, error value otherwise.
 */
int v4l2_subdev_routing_validate(struct v4l2_subdev *sd,
                                 const struct v4l2_subdev_krouting *routing,
                                 enum v4l2_subdev_routing_restriction disallow);

/**
 * v4l2_subdev_enable_streams() - Enable streams on a pad
 * @sd: The subdevice
 * @pad: The pad
 * @streams_mask: Bitmask of streams to enable
 *
 * This function enables streams on a source @pad of a subdevice. The pad is
 * identified by its index, while the streams are identified by the
 * @streams_mask bitmask. This allows enabling multiple streams on a pad at
 * once.
 *
 * Enabling a stream that is already enabled isn't allowed. If @streams_mask
 * contains an already enabled stream, this function returns -EALREADY without
 * performing any operation.
 *
 * Per-stream enable is only available for subdevs that implement the
 * .enable_streams() and .disable_streams() operations. For other subdevs, this
 * function implements a best-effort compatibility by calling the .s_stream()
 * operation, limited to subdevs that have a single source pad.
 *
 * Drivers that are not stream-aware shall set @streams_mask to BIT_ULL(0).
 *
 * Return:
 * * 0: Success
 * * -EALREADY: One of the streams in streams_mask is already enabled
 * * -EINVAL: The pad index is invalid, or doesn't correspond to a source pad
 * * -EOPNOTSUPP: Falling back to the legacy .s_stream() operation is
 *   impossible because the subdev has multiple source pads
 */
int v4l2_subdev_enable_streams(struct v4l2_subdev *sd, u32 pad,
                               u64 streams_mask);

/**
 * v4l2_subdev_disable_streams() - Disable streams on a pad
 * @sd: The subdevice
 * @pad: The pad
 * @streams_mask: Bitmask of streams to disable
 *
 * This function disables streams on a source @pad of a subdevice. The pad is
 * identified by its index, while the streams are identified by the
 * @streams_mask bitmask. This allows disabling multiple streams on a pad at
 * once.
 *
 * Disabling a streams that is not enabled isn't allowed. If @streams_mask
 * contains a disabled stream, this function returns -EALREADY without
 * performing any operation.
 *
 * Per-stream disable is only available for subdevs that implement the
 * .enable_streams() and .disable_streams() operations. For other subdevs, this
 * function implements a best-effort compatibility by calling the .s_stream()
 * operation, limited to subdevs that have a single source pad.
 *
 * Drivers that are not stream-aware shall set @streams_mask to BIT_ULL(0).
 *
 * Return:
 * * 0: Success
 * * -EALREADY: One of the streams in streams_mask is not enabled
 * * -EINVAL: The pad index is invalid, or doesn't correspond to a source pad
 * * -EOPNOTSUPP: Falling back to the legacy .s_stream() operation is
 *   impossible because the subdev has multiple source pads
 */
int v4l2_subdev_disable_streams(struct v4l2_subdev *sd, u32 pad,
                                u64 streams_mask);

/**
 * v4l2_subdev_s_stream_helper() - Helper to implement the subdev s_stream
 *        operation using enable_streams and disable_streams
 * @sd: The subdevice
 * @enable: Enable or disable streaming
 *
 * Subdevice drivers that implement the streams-aware
 * &v4l2_subdev_pad_ops.enable_streams and &v4l2_subdev_pad_ops.disable_streams
 * operations can use this helper to implement the legacy
 * &v4l2_subdev_video_ops.s_stream operation.
 *
 * This helper can only be used by subdevs that have a single source pad.
 *
 * Return: 0 on success, or a negative error code otherwise.
 */
int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable);

#endif /* CONFIG_VIDEO_V4L2_SUBDEV_API */

#endif /* CONFIG_MEDIA_CONTROLLER */

/**
 * v4l2_subdev_lock_state() - Locks the subdev state
 * @state: The subdevice state
 *
 * Locks the given subdev state.
 *
 * The state must be unlocked with v4l2_subdev_unlock_state() after use.
 */
static inline void v4l2_subdev_lock_state(struct v4l2_subdev_state *state)
{
        mutex_lock(state->lock);
}

/**
 * v4l2_subdev_unlock_state() - Unlocks the subdev state
 * @state: The subdevice state
 *
 * Unlocks the given subdev state.
 */
static inline void v4l2_subdev_unlock_state(struct v4l2_subdev_state *state)
{
        mutex_unlock(state->lock);
}

/**
 * v4l2_subdev_lock_states - Lock two sub-device states
 * @state1: One subdevice state
 * @state2: The other subdevice state
 *
 * Locks the state of two sub-devices.
 *
 * The states must be unlocked with v4l2_subdev_unlock_states() after use.
 *
 * This differs from calling v4l2_subdev_lock_state() on both states so that if
 * the states share the same lock, the lock is acquired only once (so no
 * deadlock occurs). The caller is responsible for ensuring the locks will
 * always be acquired in the same order.
 */
static inline void v4l2_subdev_lock_states(struct v4l2_subdev_state *state1,
                                           struct v4l2_subdev_state *state2)
{
        mutex_lock(state1->lock);
        if (state1->lock != state2->lock)
                mutex_lock(state2->lock);
}

/**
 * v4l2_subdev_unlock_states() - Unlock two sub-device states
 * @state1: One subdevice state
 * @state2: The other subdevice state
 *
 * Unlocks the state of two sub-devices.
 *
 * This differs from calling v4l2_subdev_unlock_state() on both states so that
 * if the states share the same lock, the lock is released only once.
 */
static inline void v4l2_subdev_unlock_states(struct v4l2_subdev_state *state1,
                                             struct v4l2_subdev_state *state2)
{
        mutex_unlock(state1->lock);
        if (state1->lock != state2->lock)
                mutex_unlock(state2->lock);
}

/**
 * v4l2_subdev_get_unlocked_active_state() - Checks that the active subdev state
 *                                             is unlocked and returns it
 * @sd: The subdevice
 *
 * Returns the active state for the subdevice, or NULL if the subdev does not
 * support active state. If the state is not NULL, calls
 * lockdep_assert_not_held() to issue a warning if the state is locked.
 *
 * This function is to be used e.g. when getting the active state for the sole
 * purpose of passing it forward, without accessing the state fields.
 */
static inline struct v4l2_subdev_state *
v4l2_subdev_get_unlocked_active_state(struct v4l2_subdev *sd)
{
        if (sd->active_state)
                lockdep_assert_not_held(sd->active_state->lock);
        return sd->active_state;
}

/**
 * v4l2_subdev_get_locked_active_state() - Checks that the active subdev state
 *                                           is locked and returns it
 *
 * @sd: The subdevice
 *
 * Returns the active state for the subdevice, or NULL if the subdev does not
 * support active state. If the state is not NULL, calls lockdep_assert_held()
 * to issue a warning if the state is not locked.
 *
 * This function is to be used when the caller knows that the active state is
 * already locked.
 */
static inline struct v4l2_subdev_state *
v4l2_subdev_get_locked_active_state(struct v4l2_subdev *sd)
{
        if (sd->active_state)
                lockdep_assert_held(sd->active_state->lock);
        return sd->active_state;
}

/**
 * v4l2_subdev_lock_and_get_active_state() - Locks and returns the active subdev
 *                                             state for the subdevice
 * @sd: The subdevice
 *
 * Returns the locked active state for the subdevice, or NULL if the subdev
 * does not support active state.
 *
 * The state must be unlocked with v4l2_subdev_unlock_state() after use.
 */
static inline struct v4l2_subdev_state *
v4l2_subdev_lock_and_get_active_state(struct v4l2_subdev *sd)
{
        if (sd->active_state)
                v4l2_subdev_lock_state(sd->active_state);
        return sd->active_state;
}

/**
 * v4l2_subdev_init - initializes the sub-device struct
 *
 * @sd: pointer to the &struct v4l2_subdev to be initialized
 * @ops: pointer to &struct v4l2_subdev_ops.
 */
void v4l2_subdev_init(struct v4l2_subdev *sd,
                      const struct v4l2_subdev_ops *ops);

extern const struct v4l2_subdev_ops v4l2_subdev_call_wrappers;

/**
 * v4l2_subdev_call - call an operation of a v4l2_subdev.
 *
 * @sd: pointer to the &struct v4l2_subdev
 * @o: name of the element at &struct v4l2_subdev_ops that contains @f.
 *     Each element there groups a set of callbacks functions.
 * @f: callback function to be called.
 *     The callback functions are defined in groups, according to
 *     each element at &struct v4l2_subdev_ops.
 * @args: arguments for @f.
 *
 * Example: err = v4l2_subdev_call(sd, video, s_std, norm);
 */
#define v4l2_subdev_call(sd, o, f, args...)                                \
        ({                                                                \
                struct v4l2_subdev *__sd = (sd);                        \
                int __result;                                                \
                if (!__sd)                                                \
                        __result = -ENODEV;                                \
                else if (!(__sd->ops->o && __sd->ops->o->f))                \
                        __result = -ENOIOCTLCMD;                        \
                else if (v4l2_subdev_call_wrappers.o &&                        \
                         v4l2_subdev_call_wrappers.o->f)                \
                        __result = v4l2_subdev_call_wrappers.o->f(        \
                                                        __sd, ##args);        \
                else                                                        \
                        __result = __sd->ops->o->f(__sd, ##args);        \
                __result;                                                \
        })

/**
 * v4l2_subdev_call_state_active - call an operation of a v4l2_subdev which
 *                                   takes state as a parameter, passing the
 *                                   subdev its active state.
 *
 * @sd: pointer to the &struct v4l2_subdev
 * @o: name of the element at &struct v4l2_subdev_ops that contains @f.
 *     Each element there groups a set of callbacks functions.
 * @f: callback function to be called.
 *     The callback functions are defined in groups, according to
 *     each element at &struct v4l2_subdev_ops.
 * @args: arguments for @f.
 *
 * This is similar to v4l2_subdev_call(), except that this version can only be
 * used for ops that take a subdev state as a parameter. The macro will get the
 * active state, lock it before calling the op and unlock it after the call.
 */
#define v4l2_subdev_call_state_active(sd, o, f, args...)                \
        ({                                                                \
                int __result;                                                \
                struct v4l2_subdev_state *state;                        \
                state = v4l2_subdev_get_unlocked_active_state(sd);        \
                if (state)                                                \
                        v4l2_subdev_lock_state(state);                        \
                __result = v4l2_subdev_call(sd, o, f, state, ##args);        \
                if (state)                                                \
                        v4l2_subdev_unlock_state(state);                \
                __result;                                                \
        })

/**
 * v4l2_subdev_call_state_try - call an operation of a v4l2_subdev which
 *                                takes state as a parameter, passing the
 *                                subdev a newly allocated try state.
 *
 * @sd: pointer to the &struct v4l2_subdev
 * @o: name of the element at &struct v4l2_subdev_ops that contains @f.
 *     Each element there groups a set of callbacks functions.
 * @f: callback function to be called.
 *     The callback functions are defined in groups, according to
 *     each element at &struct v4l2_subdev_ops.
 * @args: arguments for @f.
 *
 * This is similar to v4l2_subdev_call_state_active(), except that as this
 * version allocates a new state, this is only usable for
 * V4L2_SUBDEV_FORMAT_TRY use cases.
 *
 * Note: only legacy non-MC drivers may need this macro.
 */
#define v4l2_subdev_call_state_try(sd, o, f, args...)                 \
        ({                                                            \
                int __result;                                         \
                static struct lock_class_key __key;                   \
                const char *name = KBUILD_BASENAME                    \
                        ":" __stringify(__LINE__) ":state->lock";     \
                struct v4l2_subdev_state *state =                     \
                        __v4l2_subdev_state_alloc(sd, name, &__key);  \
                v4l2_subdev_lock_state(state);                        \
                __result = v4l2_subdev_call(sd, o, f, state, ##args); \
                v4l2_subdev_unlock_state(state);                      \
                __v4l2_subdev_state_free(state);                      \
                __result;                                             \
        })

/**
 * v4l2_subdev_has_op - Checks if a subdev defines a certain operation.
 *
 * @sd: pointer to the &struct v4l2_subdev
 * @o: The group of callback functions in &struct v4l2_subdev_ops
 * which @f is a part of.
 * @f: callback function to be checked for its existence.
 */
#define v4l2_subdev_has_op(sd, o, f) \
        ((sd)->ops->o && (sd)->ops->o->f)

/**
 * v4l2_subdev_notify_event() - Delivers event notification for subdevice
 * @sd: The subdev for which to deliver the event
 * @ev: The event to deliver
 *
 * Will deliver the specified event to all userspace event listeners which are
 * subscribed to the v42l subdev event queue as well as to the bridge driver
 * using the notify callback. The notification type for the notify callback
 * will be %V4L2_DEVICE_NOTIFY_EVENT.
 */
void v4l2_subdev_notify_event(struct v4l2_subdev *sd,
                              const struct v4l2_event *ev);

/**
 * v4l2_subdev_is_streaming() - Returns if the subdevice is streaming
 * @sd: The subdevice
 *
 * v4l2_subdev_is_streaming() tells if the subdevice is currently streaming.
 * "Streaming" here means whether .s_stream() or .enable_streams() has been
 * successfully called, and the streaming has not yet been disabled.
 *
 * If the subdevice implements .enable_streams() this function must be called
 * while holding the active state lock.
 */
bool v4l2_subdev_is_streaming(struct v4l2_subdev *sd);

#endif /* _V4L2_SUBDEV_H */











































































































  483 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Because linux/module.h has tracepoints in the header, and ftrace.h
 * used to include this file, define_trace.h includes linux/module.h
 * But we do not want the module.h to override the TRACE_SYSTEM macro
 * variable that define_trace.h is processing, so we only set it
 * when module events are being processed, which would happen when
 * CREATE_TRACE_POINTS is defined.
 */
#ifdef CREATE_TRACE_POINTS
#undef TRACE_SYSTEM
#define TRACE_SYSTEM module
#endif

#if !defined(_TRACE_MODULE_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_MODULE_H

#include <linux/tracepoint.h>

#ifdef CONFIG_MODULES

struct module;

#define show_module_flags(flags) __print_flags(flags, "",        \
        { (1UL << TAINT_PROPRIETARY_MODULE),        "P" },                \
        { (1UL << TAINT_OOT_MODULE),                "O" },                \
        { (1UL << TAINT_FORCED_MODULE),                "F" },                \
        { (1UL << TAINT_CRAP),                        "C" },                \
        { (1UL << TAINT_UNSIGNED_MODULE),        "E" })

TRACE_EVENT(module_load,

        TP_PROTO(struct module *mod),

        TP_ARGS(mod),

        TP_STRUCT__entry(
                __field(        unsigned int,        taints                )
                __string(        name,                mod->name        )
        ),

        TP_fast_assign(
                __entry->taints = mod->taints;
                __assign_str(name);
        ),

        TP_printk("%s %s", __get_str(name), show_module_flags(__entry->taints))
);

TRACE_EVENT(module_free,

        TP_PROTO(struct module *mod),

        TP_ARGS(mod),

        TP_STRUCT__entry(
                __string(        name,                mod->name        )
        ),

        TP_fast_assign(
                __assign_str(name);
        ),

        TP_printk("%s", __get_str(name))
);

#ifdef CONFIG_MODULE_UNLOAD
/* trace_module_get/put are only used if CONFIG_MODULE_UNLOAD is defined */

DECLARE_EVENT_CLASS(module_refcnt,

        TP_PROTO(struct module *mod, unsigned long ip),

        TP_ARGS(mod, ip),

        TP_STRUCT__entry(
                __field(        unsigned long,        ip                )
                __field(        int,                refcnt                )
                __string(        name,                mod->name        )
        ),

        TP_fast_assign(
                __entry->ip        = ip;
                __entry->refcnt        = atomic_read(&mod->refcnt);
                __assign_str(name);
        ),

        TP_printk("%s call_site=%ps refcnt=%d",
                  __get_str(name), (void *)__entry->ip, __entry->refcnt)
);

DEFINE_EVENT(module_refcnt, module_get,

        TP_PROTO(struct module *mod, unsigned long ip),

        TP_ARGS(mod, ip)
);

DEFINE_EVENT(module_refcnt, module_put,

        TP_PROTO(struct module *mod, unsigned long ip),

        TP_ARGS(mod, ip)
);
#endif /* CONFIG_MODULE_UNLOAD */

TRACE_EVENT(module_request,

        TP_PROTO(char *name, bool wait, unsigned long ip),

        TP_ARGS(name, wait, ip),

        TP_STRUCT__entry(
                __field(        unsigned long,        ip                )
                __field(        bool,                wait                )
                __string(        name,                name                )
        ),

        TP_fast_assign(
                __entry->ip        = ip;
                __entry->wait        = wait;
                __assign_str(name);
        ),

        TP_printk("%s wait=%d call_site=%ps",
                  __get_str(name), (int)__entry->wait, (void *)__entry->ip)
);

#endif /* CONFIG_MODULES */

#endif /* _TRACE_MODULE_H */

/* This part must be outside protection */
#include <trace/define_trace.h>




































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * (C) 2001 Clemson University and The University of Chicago
 *
 * See COPYING in top-level directory.
 */

/*
 *  The ORANGEFS Linux kernel support allows ORANGEFS volumes to be mounted and
 *  accessed through the Linux VFS (i.e. using standard I/O system calls).
 *  This support is only needed on clients that wish to mount the file system.
 *
 */

/*
 *  Declarations and macros for the ORANGEFS Linux kernel support.
 */

#ifndef __ORANGEFSKERNEL_H
#define __ORANGEFSKERNEL_H

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/statfs.h>
#include <linux/backing-dev.h>
#include <linux/device.h>
#include <linux/mpage.h>
#include <linux/namei.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/vmalloc.h>

#include <linux/aio.h>
#include <linux/posix_acl.h>
#include <linux/posix_acl_xattr.h>
#include <linux/compat.h>
#include <linux/mount.h>
#include <linux/uaccess.h>
#include <linux/atomic.h>
#include <linux/uio.h>
#include <linux/sched/signal.h>
#include <linux/mm.h>
#include <linux/wait.h>
#include <linux/dcache.h>
#include <linux/pagemap.h>
#include <linux/poll.h>
#include <linux/rwsem.h>
#include <linux/xattr.h>
#include <linux/exportfs.h>
#include <linux/hashtable.h>

#include <linux/unaligned.h>

#include "orangefs-dev-proto.h"

#define ORANGEFS_DEFAULT_OP_TIMEOUT_SECS       20

#define ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS   30

#define ORANGEFS_DEFAULT_SLOT_TIMEOUT_SECS     900        /* 15 minutes */

#define ORANGEFS_REQDEVICE_NAME          "pvfs2-req"

#define ORANGEFS_DEVREQ_MAGIC             0x20030529
#define ORANGEFS_PURGE_RETRY_COUNT     0x00000005

#define MAX_DEV_REQ_UPSIZE (2 * sizeof(__s32) +   \
sizeof(__u64) + sizeof(struct orangefs_upcall_s))
#define MAX_DEV_REQ_DOWNSIZE (2 * sizeof(__s32) + \
sizeof(__u64) + sizeof(struct orangefs_downcall_s))

/*
 * valid orangefs kernel operation states
 *
 * unknown  - op was just initialized
 * waiting  - op is on request_list (upward bound)
 * inprogr  - op is in progress (waiting for downcall)
 * serviced - op has matching downcall; ok
 * purged   - op has to start a timer since client-core
 *            exited uncleanly before servicing op
 * given up - submitter has given up waiting for it
 */
enum orangefs_vfs_op_states {
        OP_VFS_STATE_UNKNOWN = 0,
        OP_VFS_STATE_WAITING = 1,
        OP_VFS_STATE_INPROGR = 2,
        OP_VFS_STATE_SERVICED = 4,
        OP_VFS_STATE_PURGED = 8,
        OP_VFS_STATE_GIVEN_UP = 16,
};

extern const struct xattr_handler * const orangefs_xattr_handlers[];

extern struct posix_acl *orangefs_get_acl(struct inode *inode, int type, bool rcu);
extern int orangefs_set_acl(struct mnt_idmap *idmap,
                            struct dentry *dentry, struct posix_acl *acl,
                            int type);
int __orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type);

/*
 * orangefs data structures
 */
struct orangefs_kernel_op_s {
        enum orangefs_vfs_op_states op_state;
        __u64 tag;

        /*
         * Set uses_shared_memory to non zero if this operation uses
         * shared memory. If true, then a retry on the op must also
         * get a new shared memory buffer and re-populate it.
         * Cancels don't care - it only matters for service_operation()
         * retry logics and cancels don't go through it anymore. It
         * safely stays non-zero when we use it as slot_to_free.
         */
        union {
                int uses_shared_memory;
                int slot_to_free;
        };

        struct orangefs_upcall_s upcall;
        struct orangefs_downcall_s downcall;

        struct completion waitq;
        spinlock_t lock;

        int attempts;

        struct list_head list;
};

#define set_op_state_waiting(op)     ((op)->op_state = OP_VFS_STATE_WAITING)
#define set_op_state_inprogress(op)  ((op)->op_state = OP_VFS_STATE_INPROGR)
#define set_op_state_given_up(op)  ((op)->op_state = OP_VFS_STATE_GIVEN_UP)
static inline void set_op_state_serviced(struct orangefs_kernel_op_s *op)
{
        op->op_state = OP_VFS_STATE_SERVICED;
        complete(&op->waitq);
}

#define op_state_waiting(op)     ((op)->op_state & OP_VFS_STATE_WAITING)
#define op_state_in_progress(op) ((op)->op_state & OP_VFS_STATE_INPROGR)
#define op_state_serviced(op)    ((op)->op_state & OP_VFS_STATE_SERVICED)
#define op_state_purged(op)      ((op)->op_state & OP_VFS_STATE_PURGED)
#define op_state_given_up(op)    ((op)->op_state & OP_VFS_STATE_GIVEN_UP)
#define op_is_cancel(op)         ((op)->upcall.type == ORANGEFS_VFS_OP_CANCEL)

void op_release(struct orangefs_kernel_op_s *op);

extern void orangefs_bufmap_put(int);
static inline void put_cancel(struct orangefs_kernel_op_s *op)
{
        orangefs_bufmap_put(op->slot_to_free);
        op_release(op);
}

static inline void set_op_state_purged(struct orangefs_kernel_op_s *op)
{
        spin_lock(&op->lock);
        if (unlikely(op_is_cancel(op))) {
                list_del_init(&op->list);
                spin_unlock(&op->lock);
                put_cancel(op);
        } else {
                op->op_state |= OP_VFS_STATE_PURGED;
                complete(&op->waitq);
                spin_unlock(&op->lock);
        }
}

/* per inode private orangefs info */
struct orangefs_inode_s {
        struct orangefs_object_kref refn;
        char link_target[ORANGEFS_NAME_MAX];
        /*
         * Reading/Writing Extended attributes need to acquire the appropriate
         * reader/writer semaphore on the orangefs_inode_s structure.
         */
        struct rw_semaphore xattr_sem;

        struct inode vfs_inode;
        sector_t last_failed_block_index_read;

        unsigned long getattr_time;
        unsigned long mapping_time;
        int attr_valid;
        kuid_t attr_uid;
        kgid_t attr_gid;
        unsigned long bitlock;

        DECLARE_HASHTABLE(xattr_cache, 4);
};

/* per superblock private orangefs info */
struct orangefs_sb_info_s {
        struct orangefs_khandle root_khandle;
        __s32 fs_id;
        int id;
        int flags;
#define ORANGEFS_OPT_INTR        0x01
#define ORANGEFS_OPT_LOCAL_LOCK        0x02
        char devname[ORANGEFS_MAX_SERVER_ADDR_LEN];
        struct super_block *sb;
        int mount_pending;
        int no_list;
        struct list_head list;
};

struct orangefs_stats {
        unsigned long cache_hits;
        unsigned long cache_misses;
        unsigned long reads;
        unsigned long writes;
};

struct orangefs_cached_xattr {
        struct hlist_node node;
        char key[ORANGEFS_MAX_XATTR_NAMELEN];
        char val[ORANGEFS_MAX_XATTR_VALUELEN];
        ssize_t length;
        unsigned long timeout;
};

struct orangefs_write_range {
        loff_t pos;
        size_t len;
        kuid_t uid;
        kgid_t gid;
};

extern struct orangefs_stats orangefs_stats;

/*
 * NOTE: See Documentation/filesystems/porting.rst for information
 * on implementing FOO_I and properly accessing fs private data
 */
static inline struct orangefs_inode_s *ORANGEFS_I(struct inode *inode)
{
        return container_of(inode, struct orangefs_inode_s, vfs_inode);
}

static inline struct orangefs_sb_info_s *ORANGEFS_SB(struct super_block *sb)
{
        return (struct orangefs_sb_info_s *) sb->s_fs_info;
}

/* ino_t descends from "unsigned long", 8 bytes, 64 bits. */
static inline ino_t orangefs_khandle_to_ino(struct orangefs_khandle *khandle)
{
        union {
                unsigned char u[8];
                __u64 ino;
        } ihandle;

        ihandle.u[0] = khandle->u[0] ^ khandle->u[4];
        ihandle.u[1] = khandle->u[1] ^ khandle->u[5];
        ihandle.u[2] = khandle->u[2] ^ khandle->u[6];
        ihandle.u[3] = khandle->u[3] ^ khandle->u[7];
        ihandle.u[4] = khandle->u[12] ^ khandle->u[8];
        ihandle.u[5] = khandle->u[13] ^ khandle->u[9];
        ihandle.u[6] = khandle->u[14] ^ khandle->u[10];
        ihandle.u[7] = khandle->u[15] ^ khandle->u[11];

        return ihandle.ino;
}

static inline struct orangefs_khandle *get_khandle_from_ino(struct inode *inode)
{
        return &(ORANGEFS_I(inode)->refn.khandle);
}

static inline int is_root_handle(struct inode *inode)
{
        gossip_debug(GOSSIP_DCACHE_DEBUG,
                     "%s: root handle: %pU, this handle: %pU:\n",
                     __func__,
                     &ORANGEFS_SB(inode->i_sb)->root_khandle,
                     get_khandle_from_ino(inode));

        if (ORANGEFS_khandle_cmp(&(ORANGEFS_SB(inode->i_sb)->root_khandle),
                             get_khandle_from_ino(inode)))
                return 0;
        else
                return 1;
}

static inline int match_handle(struct orangefs_khandle resp_handle,
                               struct inode *inode)
{
        gossip_debug(GOSSIP_DCACHE_DEBUG,
                     "%s: one handle: %pU, another handle:%pU:\n",
                     __func__,
                     &resp_handle,
                     get_khandle_from_ino(inode));

        if (ORANGEFS_khandle_cmp(&resp_handle, get_khandle_from_ino(inode)))
                return 0;
        else
                return 1;
}

/*
 * defined in orangefs-cache.c
 */
int op_cache_initialize(void);
int op_cache_finalize(void);
struct orangefs_kernel_op_s *op_alloc(__s32 type);
void orangefs_new_tag(struct orangefs_kernel_op_s *op);
char *get_opname_string(struct orangefs_kernel_op_s *new_op);

int orangefs_inode_cache_initialize(void);
int orangefs_inode_cache_finalize(void);

/*
 * defined in orangefs-mod.c
 */
void purge_inprogress_ops(void);

/*
 * defined in waitqueue.c
 */
void purge_waiting_ops(void);

/*
 * defined in super.c
 */
extern uint64_t orangefs_features;
extern const struct fs_parameter_spec orangefs_fs_param_spec[];

int orangefs_init_fs_context(struct fs_context *fc);

void orangefs_kill_sb(struct super_block *sb);
int orangefs_remount(struct orangefs_sb_info_s *);

int fsid_key_table_initialize(void);
void fsid_key_table_finalize(void);

/*
 * defined in inode.c
 */
vm_fault_t orangefs_page_mkwrite(struct vm_fault *);
struct inode *orangefs_new_inode(struct super_block *sb,
                              struct inode *dir,
                              umode_t mode,
                              dev_t dev,
                              struct orangefs_object_kref *ref);

int __orangefs_setattr(struct inode *, struct iattr *);
int __orangefs_setattr_mode(struct dentry *dentry, struct iattr *iattr);
int orangefs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *);

int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path,
                     struct kstat *stat, u32 request_mask, unsigned int flags);

int orangefs_permission(struct mnt_idmap *idmap,
                        struct inode *inode, int mask);

int orangefs_update_time(struct inode *, int);

/*
 * defined in xattr.c
 */
ssize_t orangefs_listxattr(struct dentry *dentry, char *buffer, size_t size);

/*
 * defined in namei.c
 */
struct inode *orangefs_iget(struct super_block *sb,
                         struct orangefs_object_kref *ref);

/*
 * defined in devorangefs-req.c
 */
extern uint32_t orangefs_userspace_version;

int orangefs_dev_init(void);
void orangefs_dev_cleanup(void);
int is_daemon_in_service(void);
bool __is_daemon_in_service(void);

/*
 * defined in file.c
 */
int orangefs_revalidate_mapping(struct inode *);
ssize_t wait_for_direct_io(enum ORANGEFS_io_type, struct inode *, loff_t *,
    struct iov_iter *, size_t, loff_t, struct orangefs_write_range *, int *,
    struct file *);
ssize_t do_readv_writev(enum ORANGEFS_io_type, struct file *, loff_t *,
    struct iov_iter *);

/*
 * defined in orangefs-utils.c
 */
__s32 fsid_of_op(struct orangefs_kernel_op_s *op);

ssize_t orangefs_inode_getxattr(struct inode *inode,
                             const char *name,
                             void *buffer,
                             size_t size);

int orangefs_inode_setxattr(struct inode *inode,
                         const char *name,
                         const void *value,
                         size_t size,
                         int flags);

#define ORANGEFS_GETATTR_NEW 1
#define ORANGEFS_GETATTR_SIZE 2

int orangefs_inode_getattr(struct inode *, int);

int orangefs_inode_check_changed(struct inode *inode);

int orangefs_inode_setattr(struct inode *inode);

bool orangefs_cancel_op_in_progress(struct orangefs_kernel_op_s *op);

int orangefs_normalize_to_errno(__s32 error_code);

extern struct mutex orangefs_request_mutex;
extern int op_timeout_secs;
extern int slot_timeout_secs;
extern int orangefs_cache_timeout_msecs;
extern int orangefs_dcache_timeout_msecs;
extern int orangefs_getattr_timeout_msecs;
extern struct list_head orangefs_superblocks;
extern spinlock_t orangefs_superblocks_lock;
extern struct list_head orangefs_request_list;
extern spinlock_t orangefs_request_list_lock;
extern wait_queue_head_t orangefs_request_list_waitq;
extern struct list_head *orangefs_htable_ops_in_progress;
extern spinlock_t orangefs_htable_ops_in_progress_lock;
extern int hash_table_size;

extern const struct file_operations orangefs_file_operations;
extern const struct inode_operations orangefs_symlink_inode_operations;
extern const struct inode_operations orangefs_dir_inode_operations;
extern const struct file_operations orangefs_dir_operations;
extern const struct dentry_operations orangefs_dentry_operations;

/*
 * misc convenience macros
 */

#define ORANGEFS_OP_INTERRUPTIBLE 1   /* service_operation() is interruptible */
#define ORANGEFS_OP_PRIORITY      2   /* service_operation() is high priority */
#define ORANGEFS_OP_CANCELLATION  4   /* this is a cancellation */
#define ORANGEFS_OP_NO_MUTEX      8   /* don't acquire request_mutex */
#define ORANGEFS_OP_ASYNC         16  /* Queue it, but don't wait */
#define ORANGEFS_OP_WRITEBACK     32

int service_operation(struct orangefs_kernel_op_s *op,
                      const char *op_name,
                      int flags);

#define get_interruptible_flag(inode) \
        ((ORANGEFS_SB(inode->i_sb)->flags & ORANGEFS_OPT_INTR) ? \
                ORANGEFS_OP_INTERRUPTIBLE : 0)

#define fill_default_sys_attrs(sys_attr, type, mode)                        \
do {                                                                        \
        sys_attr.owner = from_kuid(&init_user_ns, current_fsuid()); \
        sys_attr.group = from_kgid(&init_user_ns, current_fsgid()); \
        sys_attr.perms = ORANGEFS_util_translate_mode(mode);                \
        sys_attr.mtime = 0;                                                \
        sys_attr.atime = 0;                                                \
        sys_attr.ctime = 0;                                                \
        sys_attr.mask = ORANGEFS_ATTR_SYS_ALL_SETABLE;                        \
} while (0)

static inline void orangefs_set_timeout(struct dentry *dentry)
{
        unsigned long time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000;

        dentry->d_fsdata = (void *) time;
}

#endif /* __ORANGEFSKERNEL_H */

































































































   73 











































































































   35 
  101 









  273 






   83 




  321 































































































   85 


   84 








  116 





   91 





























   85 







   85 



   85 





   85 















  149 







  149 
  149 


  149 



  116 

   91 


  127 
  128 

   43 

  149 


  148 
   60 





   85 



   85 
   84 

   85 






   85 
   42 

   85 







   89 

   83 


















   85 

   85 





   85 








   83 
   85 




   85 





   85 


   85 
   84 


   85 
   42 












   85 













   85 
   84 




   68 




   83 








   33 
   33 




























   42 
   42 
   42 
   42 
   42 


























   32 













   32 
   32 
   32 




   32 




   32 





























   61 
   61 


   61 
   61 
   61 








  217 















  216 
  216 

   39 



  217 








































































  267 









  234 




  245 








   61 




  244 




  245 








  265 



  266 
  265 


  267 






  265 




  267 
   85 








   84 



  266 


   84 





  247 
   32 








   32 



  246 
   61 


  246 



   56 


   40 

   19 









   97 

   15 






  231 





  231 

  145 


  228 















  231 





  231 




















  212 




    6 

   70 

  211 

    2 





    1 
























    1 
    1 









    2 





    2 

    1 







    2 















    2 






















    2 





  148 













   97 







   84 


   58 





   57 


   58 







   96 

   97 
   97 











  144 

   97 


    1 



   83 
   79 

   85 





    1 
   84 

    7 



   87 

    4 











  236 













  236 



  121 





  119 











    2 




    1 











   51 

    1 




   50 













    3 





















    4 

    1 







    4 












    3 
    1 







   47 


   47 





    7 

























   58 
   65 




   52 










  100 
   10 


























  949 



  956 




  949 








 1146 













 1150 

    6 









 1141 















 1149 



  737 










 1144 
 1145 












  381 



  310 







  309 


  400 
  443 






  303 







  329 







   13 
   13 















   42 





  953 


  953 




    1 

  951 
  951 
    5 























  956 
  946 




  941 






























   80 





   80 





   32 
   53 
   68 

   69 




   36 



    6 














  176 







   97 













  100 











   69 



   92 



   98 



   86 


    3 
    1 



   97 
    6 
   89 
   25 
   71 
   21 
   74 
   18 









   80 



   79 
    1 



   42 
   69 








   59 







   59 

   59 



   59 




   59 




   56 

   59 


   58 
   58 




   59 
   59 



   59 










































































   37 







   13 








   32 


   32 


   32 










   32 
   32 

   32 





















   32 



















   32 
   32 



   32 





   32 
   32 





   32 




   32 




   32 





   32 





   32 
   32 



   32 



  167 





















  167 
  167 



  167 





  165 
  167 





  164 




  167 


  167 
  166 
  167 







  167 
    4 



  161 

  166 
    1 

  166 








  167 
   15 




  165 










































































    2 








    1 


    2 




















    2 
    2 





    2 







    2 



    2 



    2 
    2 




















   32 












   11 
    8 







   19 


    5 




   18 


    8 
   11 


    9 



    7 




    2 























    5 


    9 







   19 











   19 














   18 


   19 



































   73 















    2 







   73 



















    4 




    5 


    5 
    5 
    5 

    5 



    5 



    5 









    5 
    5 



















    5 
    1 


    4 



















    1 
    1 





    1 
    1 

    1 




    1 



    1 










    1 

















    1 



    1 
    1 
    1 


















    1 
























    5 

    5 






    1 








    1 
    1 



    1 






    1 

    1 
    1 

    1 


    1 

    1 











    4 



    4 


    4 


    4 











    4 
    4 
    4 



    4 










    4 




    4 







    1 


    4 













    4 
    4 




    4 








    4 
    4 
























    4 







    4 



    4 
    4 




    4 








    4 


    4 


    4 


    4 

    4 

    4 
































    3 

    3 





    3 





    3 




    3 








    6 




    6 


    1 







    4 

    6 






    6 










    3 
    6 










    6 
    6 









    6 

















    6 







    6 







    6 

    6 
    6 

    6 



    6 





    6 


    6 


































   40 








   40 





















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *
 *   Robert Olsson <robert.olsson@its.uu.se> Uppsala Universitet
 *     & Swedish University of Agricultural Sciences.
 *
 *   Jens Laas <jens.laas@data.slu.se> Swedish University of
 *     Agricultural Sciences.
 *
 *   Hans Liss <hans.liss@its.uu.se>  Uppsala Universitet
 *
 * This work is based on the LPC-trie which is originally described in:
 *
 * An experimental study of compression methods for dynamic tries
 * Stefan Nilsson and Matti Tikkanen. Algorithmica, 33(1):19-33, 2002.
 * https://www.csc.kth.se/~snilsson/software/dyntrie2/
 *
 * IP-address lookup using LC-tries. Stefan Nilsson and Gunnar Karlsson
 * IEEE Journal on Selected Areas in Communications, 17(6):1083-1092, June 1999
 *
 * Code from fib_hash has been reused which includes the following header:
 *
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                IPv4 FIB: lookup engine and maintenance routines.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Substantial contributions to this work comes from:
 *
 *                David S. Miller, <davem@davemloft.net>
 *                Stephen Hemminger <shemminger@osdl.org>
 *                Paul E. McKenney <paulmck@us.ibm.com>
 *                Patrick McHardy <kaber@trash.net>
 */
#include <linux/cache.h>
#include <linux/uaccess.h>
#include <linux/bitops.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/string.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/errno.h>
#include <linux/in.h>
#include <linux/inet.h>
#include <linux/inetdevice.h>
#include <linux/netdevice.h>
#include <linux/if_arp.h>
#include <linux/proc_fs.h>
#include <linux/rcupdate.h>
#include <linux/rcupdate_wait.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/init.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/vmalloc.h>
#include <linux/notifier.h>
#include <net/net_namespace.h>
#include <net/inet_dscp.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/route.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/ip_fib.h>
#include <net/fib_notifier.h>
#include <trace/events/fib.h>
#include "fib_lookup.h"

static int call_fib_entry_notifier(struct notifier_block *nb,
                                   enum fib_event_type event_type, u32 dst,
                                   int dst_len, struct fib_alias *fa,
                                   struct netlink_ext_ack *extack)
{
        struct fib_entry_notifier_info info = {
                .info.extack = extack,
                .dst = dst,
                .dst_len = dst_len,
                .fi = fa->fa_info,
                .dscp = fa->fa_dscp,
                .type = fa->fa_type,
                .tb_id = fa->tb_id,
        };
        return call_fib4_notifier(nb, event_type, &info.info);
}

static int call_fib_entry_notifiers(struct net *net,
                                    enum fib_event_type event_type, u32 dst,
                                    int dst_len, struct fib_alias *fa,
                                    struct netlink_ext_ack *extack)
{
        struct fib_entry_notifier_info info = {
                .info.extack = extack,
                .dst = dst,
                .dst_len = dst_len,
                .fi = fa->fa_info,
                .dscp = fa->fa_dscp,
                .type = fa->fa_type,
                .tb_id = fa->tb_id,
        };
        return call_fib4_notifiers(net, event_type, &info.info);
}

#define MAX_STAT_DEPTH 32

#define KEYLENGTH        (8*sizeof(t_key))
#define KEY_MAX                ((t_key)~0)

typedef unsigned int t_key;

#define IS_TRIE(n)        ((n)->pos >= KEYLENGTH)
#define IS_TNODE(n)        ((n)->bits)
#define IS_LEAF(n)        (!(n)->bits)

struct key_vector {
        t_key key;
        unsigned char pos;                /* 2log(KEYLENGTH) bits needed */
        unsigned char bits;                /* 2log(KEYLENGTH) bits needed */
        unsigned char slen;
        union {
                /* This list pointer if valid if (pos | bits) == 0 (LEAF) */
                struct hlist_head leaf;
                /* This array is valid if (pos | bits) > 0 (TNODE) */
                DECLARE_FLEX_ARRAY(struct key_vector __rcu *, tnode);
        };
};

struct tnode {
        struct rcu_head rcu;
        t_key empty_children;                /* KEYLENGTH bits needed */
        t_key full_children;                /* KEYLENGTH bits needed */
        struct key_vector __rcu *parent;
        struct key_vector kv[1];
#define tn_bits kv[0].bits
};

#define TNODE_SIZE(n)        offsetof(struct tnode, kv[0].tnode[n])
#define LEAF_SIZE        TNODE_SIZE(1)

#ifdef CONFIG_IP_FIB_TRIE_STATS
struct trie_use_stats {
        unsigned int gets;
        unsigned int backtrack;
        unsigned int semantic_match_passed;
        unsigned int semantic_match_miss;
        unsigned int null_node_hit;
        unsigned int resize_node_skipped;
};
#endif

struct trie_stat {
        unsigned int totdepth;
        unsigned int maxdepth;
        unsigned int tnodes;
        unsigned int leaves;
        unsigned int nullpointers;
        unsigned int prefixes;
        unsigned int nodesizes[MAX_STAT_DEPTH];
};

struct trie {
        struct key_vector kv[1];
#ifdef CONFIG_IP_FIB_TRIE_STATS
        struct trie_use_stats __percpu *stats;
#endif
};

static struct key_vector *resize(struct trie *t, struct key_vector *tn);
static unsigned int tnode_free_size;

/*
 * synchronize_rcu after call_rcu for outstanding dirty memory; it should be
 * especially useful before resizing the root node with PREEMPT_NONE configs;
 * the value was obtained experimentally, aiming to avoid visible slowdown.
 */
unsigned int sysctl_fib_sync_mem = 512 * 1024;
unsigned int sysctl_fib_sync_mem_min = 64 * 1024;
unsigned int sysctl_fib_sync_mem_max = 64 * 1024 * 1024;

static struct kmem_cache *fn_alias_kmem __ro_after_init;
static struct kmem_cache *trie_leaf_kmem __ro_after_init;

static inline struct tnode *tn_info(struct key_vector *kv)
{
        return container_of(kv, struct tnode, kv[0]);
}

/* caller must hold RTNL */
#define node_parent(tn) rtnl_dereference(tn_info(tn)->parent)
#define get_child(tn, i) rtnl_dereference((tn)->tnode[i])

/* caller must hold RCU read lock or RTNL */
#define node_parent_rcu(tn) rcu_dereference_rtnl(tn_info(tn)->parent)
#define get_child_rcu(tn, i) rcu_dereference_rtnl((tn)->tnode[i])

/* wrapper for rcu_assign_pointer */
static inline void node_set_parent(struct key_vector *n, struct key_vector *tp)
{
        if (n)
                rcu_assign_pointer(tn_info(n)->parent, tp);
}

#define NODE_INIT_PARENT(n, p) RCU_INIT_POINTER(tn_info(n)->parent, p)

/* This provides us with the number of children in this node, in the case of a
 * leaf this will return 0 meaning none of the children are accessible.
 */
static inline unsigned long child_length(const struct key_vector *tn)
{
        return (1ul << tn->bits) & ~(1ul);
}

#define get_cindex(key, kv) (((key) ^ (kv)->key) >> (kv)->pos)

static inline unsigned long get_index(t_key key, struct key_vector *kv)
{
        unsigned long index = key ^ kv->key;

        if ((BITS_PER_LONG <= KEYLENGTH) && (KEYLENGTH == kv->pos))
                return 0;

        return index >> kv->pos;
}

/* To understand this stuff, an understanding of keys and all their bits is
 * necessary. Every node in the trie has a key associated with it, but not
 * all of the bits in that key are significant.
 *
 * Consider a node 'n' and its parent 'tp'.
 *
 * If n is a leaf, every bit in its key is significant. Its presence is
 * necessitated by path compression, since during a tree traversal (when
 * searching for a leaf - unless we are doing an insertion) we will completely
 * ignore all skipped bits we encounter. Thus we need to verify, at the end of
 * a potentially successful search, that we have indeed been walking the
 * correct key path.
 *
 * Note that we can never "miss" the correct key in the tree if present by
 * following the wrong path. Path compression ensures that segments of the key
 * that are the same for all keys with a given prefix are skipped, but the
 * skipped part *is* identical for each node in the subtrie below the skipped
 * bit! trie_insert() in this implementation takes care of that.
 *
 * if n is an internal node - a 'tnode' here, the various parts of its key
 * have many different meanings.
 *
 * Example:
 * _________________________________________________________________
 * | i | i | i | i | i | i | i | N | N | N | S | S | S | S | S | C |
 * -----------------------------------------------------------------
 *  31  30  29  28  27  26  25  24  23  22  21  20  19  18  17  16
 *
 * _________________________________________________________________
 * | C | C | C | u | u | u | u | u | u | u | u | u | u | u | u | u |
 * -----------------------------------------------------------------
 *  15  14  13  12  11  10   9   8   7   6   5   4   3   2   1   0
 *
 * tp->pos = 22
 * tp->bits = 3
 * n->pos = 13
 * n->bits = 4
 *
 * First, let's just ignore the bits that come before the parent tp, that is
 * the bits from (tp->pos + tp->bits) to 31. They are *known* but at this
 * point we do not use them for anything.
 *
 * The bits from (tp->pos) to (tp->pos + tp->bits - 1) - "N", above - are the
 * index into the parent's child array. That is, they will be used to find
 * 'n' among tp's children.
 *
 * The bits from (n->pos + n->bits) to (tp->pos - 1) - "S" - are skipped bits
 * for the node n.
 *
 * All the bits we have seen so far are significant to the node n. The rest
 * of the bits are really not needed or indeed known in n->key.
 *
 * The bits from (n->pos) to (n->pos + n->bits - 1) - "C" - are the index into
 * n's child array, and will of course be different for each child.
 *
 * The rest of the bits, from 0 to (n->pos -1) - "u" - are completely unknown
 * at this point.
 */

static const int halve_threshold = 25;
static const int inflate_threshold = 50;
static const int halve_threshold_root = 15;
static const int inflate_threshold_root = 30;

static inline void alias_free_mem_rcu(struct fib_alias *fa)
{
        kfree_rcu(fa, rcu);
}

#define TNODE_VMALLOC_MAX \
        ilog2((SIZE_MAX - TNODE_SIZE(0)) / sizeof(struct key_vector *))

static void __node_free_rcu(struct rcu_head *head)
{
        struct tnode *n = container_of(head, struct tnode, rcu);

        if (!n->tn_bits)
                kmem_cache_free(trie_leaf_kmem, n);
        else
                kvfree(n);
}

#define node_free(n) call_rcu(&tn_info(n)->rcu, __node_free_rcu)

static struct tnode *tnode_alloc(int bits)
{
        size_t size;

        /* verify bits is within bounds */
        if (bits > TNODE_VMALLOC_MAX)
                return NULL;

        /* determine size and verify it is non-zero and didn't overflow */
        size = TNODE_SIZE(1ul << bits);

        if (size <= PAGE_SIZE)
                return kzalloc(size, GFP_KERNEL);
        else
                return vzalloc(size);
}

static inline void empty_child_inc(struct key_vector *n)
{
        tn_info(n)->empty_children++;

        if (!tn_info(n)->empty_children)
                tn_info(n)->full_children++;
}

static inline void empty_child_dec(struct key_vector *n)
{
        if (!tn_info(n)->empty_children)
                tn_info(n)->full_children--;

        tn_info(n)->empty_children--;
}

static struct key_vector *leaf_new(t_key key, struct fib_alias *fa)
{
        struct key_vector *l;
        struct tnode *kv;

        kv = kmem_cache_alloc(trie_leaf_kmem, GFP_KERNEL);
        if (!kv)
                return NULL;

        /* initialize key vector */
        l = kv->kv;
        l->key = key;
        l->pos = 0;
        l->bits = 0;
        l->slen = fa->fa_slen;

        /* link leaf to fib alias */
        INIT_HLIST_HEAD(&l->leaf);
        hlist_add_head(&fa->fa_list, &l->leaf);

        return l;
}

static struct key_vector *tnode_new(t_key key, int pos, int bits)
{
        unsigned int shift = pos + bits;
        struct key_vector *tn;
        struct tnode *tnode;

        /* verify bits and pos their msb bits clear and values are valid */
        BUG_ON(!bits || (shift > KEYLENGTH));

        tnode = tnode_alloc(bits);
        if (!tnode)
                return NULL;

        pr_debug("AT %p s=%zu %zu\n", tnode, TNODE_SIZE(0),
                 sizeof(struct key_vector *) << bits);

        if (bits == KEYLENGTH)
                tnode->full_children = 1;
        else
                tnode->empty_children = 1ul << bits;

        tn = tnode->kv;
        tn->key = (shift < KEYLENGTH) ? (key >> shift) << shift : 0;
        tn->pos = pos;
        tn->bits = bits;
        tn->slen = pos;

        return tn;
}

/* Check whether a tnode 'n' is "full", i.e. it is an internal node
 * and no bits are skipped. See discussion in dyntree paper p. 6
 */
static inline int tnode_full(struct key_vector *tn, struct key_vector *n)
{
        return n && ((n->pos + n->bits) == tn->pos) && IS_TNODE(n);
}

/* Add a child at position i overwriting the old value.
 * Update the value of full_children and empty_children.
 */
static void put_child(struct key_vector *tn, unsigned long i,
                      struct key_vector *n)
{
        struct key_vector *chi = get_child(tn, i);
        int isfull, wasfull;

        BUG_ON(i >= child_length(tn));

        /* update emptyChildren, overflow into fullChildren */
        if (!n && chi)
                empty_child_inc(tn);
        if (n && !chi)
                empty_child_dec(tn);

        /* update fullChildren */
        wasfull = tnode_full(tn, chi);
        isfull = tnode_full(tn, n);

        if (wasfull && !isfull)
                tn_info(tn)->full_children--;
        else if (!wasfull && isfull)
                tn_info(tn)->full_children++;

        if (n && (tn->slen < n->slen))
                tn->slen = n->slen;

        rcu_assign_pointer(tn->tnode[i], n);
}

static void update_children(struct key_vector *tn)
{
        unsigned long i;

        /* update all of the child parent pointers */
        for (i = child_length(tn); i;) {
                struct key_vector *inode = get_child(tn, --i);

                if (!inode)
                        continue;

                /* Either update the children of a tnode that
                 * already belongs to us or update the child
                 * to point to ourselves.
                 */
                if (node_parent(inode) == tn)
                        update_children(inode);
                else
                        node_set_parent(inode, tn);
        }
}

static inline void put_child_root(struct key_vector *tp, t_key key,
                                  struct key_vector *n)
{
        if (IS_TRIE(tp))
                rcu_assign_pointer(tp->tnode[0], n);
        else
                put_child(tp, get_index(key, tp), n);
}

static inline void tnode_free_init(struct key_vector *tn)
{
        tn_info(tn)->rcu.next = NULL;
}

static inline void tnode_free_append(struct key_vector *tn,
                                     struct key_vector *n)
{
        tn_info(n)->rcu.next = tn_info(tn)->rcu.next;
        tn_info(tn)->rcu.next = &tn_info(n)->rcu;
}

static void tnode_free(struct key_vector *tn)
{
        struct callback_head *head = &tn_info(tn)->rcu;

        while (head) {
                head = head->next;
                tnode_free_size += TNODE_SIZE(1ul << tn->bits);
                node_free(tn);

                tn = container_of(head, struct tnode, rcu)->kv;
        }

        if (tnode_free_size >= READ_ONCE(sysctl_fib_sync_mem)) {
                tnode_free_size = 0;
                synchronize_net();
        }
}

static struct key_vector *replace(struct trie *t,
                                  struct key_vector *oldtnode,
                                  struct key_vector *tn)
{
        struct key_vector *tp = node_parent(oldtnode);
        unsigned long i;

        /* setup the parent pointer out of and back into this node */
        NODE_INIT_PARENT(tn, tp);
        put_child_root(tp, tn->key, tn);

        /* update all of the child parent pointers */
        update_children(tn);

        /* all pointers should be clean so we are done */
        tnode_free(oldtnode);

        /* resize children now that oldtnode is freed */
        for (i = child_length(tn); i;) {
                struct key_vector *inode = get_child(tn, --i);

                /* resize child node */
                if (tnode_full(tn, inode))
                        tn = resize(t, inode);
        }

        return tp;
}

static struct key_vector *inflate(struct trie *t,
                                  struct key_vector *oldtnode)
{
        struct key_vector *tn;
        unsigned long i;
        t_key m;

        pr_debug("In inflate\n");

        tn = tnode_new(oldtnode->key, oldtnode->pos - 1, oldtnode->bits + 1);
        if (!tn)
                goto notnode;

        /* prepare oldtnode to be freed */
        tnode_free_init(oldtnode);

        /* Assemble all of the pointers in our cluster, in this case that
         * represents all of the pointers out of our allocated nodes that
         * point to existing tnodes and the links between our allocated
         * nodes.
         */
        for (i = child_length(oldtnode), m = 1u << tn->pos; i;) {
                struct key_vector *inode = get_child(oldtnode, --i);
                struct key_vector *node0, *node1;
                unsigned long j, k;

                /* An empty child */
                if (!inode)
                        continue;

                /* A leaf or an internal node with skipped bits */
                if (!tnode_full(oldtnode, inode)) {
                        put_child(tn, get_index(inode->key, tn), inode);
                        continue;
                }

                /* drop the node in the old tnode free list */
                tnode_free_append(oldtnode, inode);

                /* An internal node with two children */
                if (inode->bits == 1) {
                        put_child(tn, 2 * i + 1, get_child(inode, 1));
                        put_child(tn, 2 * i, get_child(inode, 0));
                        continue;
                }

                /* We will replace this node 'inode' with two new
                 * ones, 'node0' and 'node1', each with half of the
                 * original children. The two new nodes will have
                 * a position one bit further down the key and this
                 * means that the "significant" part of their keys
                 * (see the discussion near the top of this file)
                 * will differ by one bit, which will be "0" in
                 * node0's key and "1" in node1's key. Since we are
                 * moving the key position by one step, the bit that
                 * we are moving away from - the bit at position
                 * (tn->pos) - is the one that will differ between
                 * node0 and node1. So... we synthesize that bit in the
                 * two new keys.
                 */
                node1 = tnode_new(inode->key | m, inode->pos, inode->bits - 1);
                if (!node1)
                        goto nomem;
                node0 = tnode_new(inode->key, inode->pos, inode->bits - 1);

                tnode_free_append(tn, node1);
                if (!node0)
                        goto nomem;
                tnode_free_append(tn, node0);

                /* populate child pointers in new nodes */
                for (k = child_length(inode), j = k / 2; j;) {
                        put_child(node1, --j, get_child(inode, --k));
                        put_child(node0, j, get_child(inode, j));
                        put_child(node1, --j, get_child(inode, --k));
                        put_child(node0, j, get_child(inode, j));
                }

                /* link new nodes to parent */
                NODE_INIT_PARENT(node1, tn);
                NODE_INIT_PARENT(node0, tn);

                /* link parent to nodes */
                put_child(tn, 2 * i + 1, node1);
                put_child(tn, 2 * i, node0);
        }

        /* setup the parent pointers into and out of this node */
        return replace(t, oldtnode, tn);
nomem:
        /* all pointers should be clean so we are done */
        tnode_free(tn);
notnode:
        return NULL;
}

static struct key_vector *halve(struct trie *t,
                                struct key_vector *oldtnode)
{
        struct key_vector *tn;
        unsigned long i;

        pr_debug("In halve\n");

        tn = tnode_new(oldtnode->key, oldtnode->pos + 1, oldtnode->bits - 1);
        if (!tn)
                goto notnode;

        /* prepare oldtnode to be freed */
        tnode_free_init(oldtnode);

        /* Assemble all of the pointers in our cluster, in this case that
         * represents all of the pointers out of our allocated nodes that
         * point to existing tnodes and the links between our allocated
         * nodes.
         */
        for (i = child_length(oldtnode); i;) {
                struct key_vector *node1 = get_child(oldtnode, --i);
                struct key_vector *node0 = get_child(oldtnode, --i);
                struct key_vector *inode;

                /* At least one of the children is empty */
                if (!node1 || !node0) {
                        put_child(tn, i / 2, node1 ? : node0);
                        continue;
                }

                /* Two nonempty children */
                inode = tnode_new(node0->key, oldtnode->pos, 1);
                if (!inode)
                        goto nomem;
                tnode_free_append(tn, inode);

                /* initialize pointers out of node */
                put_child(inode, 1, node1);
                put_child(inode, 0, node0);
                NODE_INIT_PARENT(inode, tn);

                /* link parent to node */
                put_child(tn, i / 2, inode);
        }

        /* setup the parent pointers into and out of this node */
        return replace(t, oldtnode, tn);
nomem:
        /* all pointers should be clean so we are done */
        tnode_free(tn);
notnode:
        return NULL;
}

static struct key_vector *collapse(struct trie *t,
                                   struct key_vector *oldtnode)
{
        struct key_vector *n, *tp;
        unsigned long i;

        /* scan the tnode looking for that one child that might still exist */
        for (n = NULL, i = child_length(oldtnode); !n && i;)
                n = get_child(oldtnode, --i);

        /* compress one level */
        tp = node_parent(oldtnode);
        put_child_root(tp, oldtnode->key, n);
        node_set_parent(n, tp);

        /* drop dead node */
        node_free(oldtnode);

        return tp;
}

static unsigned char update_suffix(struct key_vector *tn)
{
        unsigned char slen = tn->pos;
        unsigned long stride, i;
        unsigned char slen_max;

        /* only vector 0 can have a suffix length greater than or equal to
         * tn->pos + tn->bits, the second highest node will have a suffix
         * length at most of tn->pos + tn->bits - 1
         */
        slen_max = min_t(unsigned char, tn->pos + tn->bits - 1, tn->slen);

        /* search though the list of children looking for nodes that might
         * have a suffix greater than the one we currently have.  This is
         * why we start with a stride of 2 since a stride of 1 would
         * represent the nodes with suffix length equal to tn->pos
         */
        for (i = 0, stride = 0x2ul ; i < child_length(tn); i += stride) {
                struct key_vector *n = get_child(tn, i);

                if (!n || (n->slen <= slen))
                        continue;

                /* update stride and slen based on new value */
                stride <<= (n->slen - slen);
                slen = n->slen;
                i &= ~(stride - 1);

                /* stop searching if we have hit the maximum possible value */
                if (slen >= slen_max)
                        break;
        }

        tn->slen = slen;

        return slen;
}

/* From "Implementing a dynamic compressed trie" by Stefan Nilsson of
 * the Helsinki University of Technology and Matti Tikkanen of Nokia
 * Telecommunications, page 6:
 * "A node is doubled if the ratio of non-empty children to all
 * children in the *doubled* node is at least 'high'."
 *
 * 'high' in this instance is the variable 'inflate_threshold'. It
 * is expressed as a percentage, so we multiply it with
 * child_length() and instead of multiplying by 2 (since the
 * child array will be doubled by inflate()) and multiplying
 * the left-hand side by 100 (to handle the percentage thing) we
 * multiply the left-hand side by 50.
 *
 * The left-hand side may look a bit weird: child_length(tn)
 * - tn->empty_children is of course the number of non-null children
 * in the current node. tn->full_children is the number of "full"
 * children, that is non-null tnodes with a skip value of 0.
 * All of those will be doubled in the resulting inflated tnode, so
 * we just count them one extra time here.
 *
 * A clearer way to write this would be:
 *
 * to_be_doubled = tn->full_children;
 * not_to_be_doubled = child_length(tn) - tn->empty_children -
 *     tn->full_children;
 *
 * new_child_length = child_length(tn) * 2;
 *
 * new_fill_factor = 100 * (not_to_be_doubled + 2*to_be_doubled) /
 *      new_child_length;
 * if (new_fill_factor >= inflate_threshold)
 *
 * ...and so on, tho it would mess up the while () loop.
 *
 * anyway,
 * 100 * (not_to_be_doubled + 2*to_be_doubled) / new_child_length >=
 *      inflate_threshold
 *
 * avoid a division:
 * 100 * (not_to_be_doubled + 2*to_be_doubled) >=
 *      inflate_threshold * new_child_length
 *
 * expand not_to_be_doubled and to_be_doubled, and shorten:
 * 100 * (child_length(tn) - tn->empty_children +
 *    tn->full_children) >= inflate_threshold * new_child_length
 *
 * expand new_child_length:
 * 100 * (child_length(tn) - tn->empty_children +
 *    tn->full_children) >=
 *      inflate_threshold * child_length(tn) * 2
 *
 * shorten again:
 * 50 * (tn->full_children + child_length(tn) -
 *    tn->empty_children) >= inflate_threshold *
 *    child_length(tn)
 *
 */
static inline bool should_inflate(struct key_vector *tp, struct key_vector *tn)
{
        unsigned long used = child_length(tn);
        unsigned long threshold = used;

        /* Keep root node larger */
        threshold *= IS_TRIE(tp) ? inflate_threshold_root : inflate_threshold;
        used -= tn_info(tn)->empty_children;
        used += tn_info(tn)->full_children;

        /* if bits == KEYLENGTH then pos = 0, and will fail below */

        return (used > 1) && tn->pos && ((50 * used) >= threshold);
}

static inline bool should_halve(struct key_vector *tp, struct key_vector *tn)
{
        unsigned long used = child_length(tn);
        unsigned long threshold = used;

        /* Keep root node larger */
        threshold *= IS_TRIE(tp) ? halve_threshold_root : halve_threshold;
        used -= tn_info(tn)->empty_children;

        /* if bits == KEYLENGTH then used = 100% on wrap, and will fail below */

        return (used > 1) && (tn->bits > 1) && ((100 * used) < threshold);
}

static inline bool should_collapse(struct key_vector *tn)
{
        unsigned long used = child_length(tn);

        used -= tn_info(tn)->empty_children;

        /* account for bits == KEYLENGTH case */
        if ((tn->bits == KEYLENGTH) && tn_info(tn)->full_children)
                used -= KEY_MAX;

        /* One child or none, time to drop us from the trie */
        return used < 2;
}

#define MAX_WORK 10
static struct key_vector *resize(struct trie *t, struct key_vector *tn)
{
#ifdef CONFIG_IP_FIB_TRIE_STATS
        struct trie_use_stats __percpu *stats = t->stats;
#endif
        struct key_vector *tp = node_parent(tn);
        unsigned long cindex = get_index(tn->key, tp);
        int max_work = MAX_WORK;

        pr_debug("In tnode_resize %p inflate_threshold=%d threshold=%d\n",
                 tn, inflate_threshold, halve_threshold);

        /* track the tnode via the pointer from the parent instead of
         * doing it ourselves.  This way we can let RCU fully do its
         * thing without us interfering
         */
        BUG_ON(tn != get_child(tp, cindex));

        /* Double as long as the resulting node has a number of
         * nonempty nodes that are above the threshold.
         */
        while (should_inflate(tp, tn) && max_work) {
                tp = inflate(t, tn);
                if (!tp) {
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        this_cpu_inc(stats->resize_node_skipped);
#endif
                        break;
                }

                max_work--;
                tn = get_child(tp, cindex);
        }

        /* update parent in case inflate failed */
        tp = node_parent(tn);

        /* Return if at least one inflate is run */
        if (max_work != MAX_WORK)
                return tp;

        /* Halve as long as the number of empty children in this
         * node is above threshold.
         */
        while (should_halve(tp, tn) && max_work) {
                tp = halve(t, tn);
                if (!tp) {
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        this_cpu_inc(stats->resize_node_skipped);
#endif
                        break;
                }

                max_work--;
                tn = get_child(tp, cindex);
        }

        /* Only one child remains */
        if (should_collapse(tn))
                return collapse(t, tn);

        /* update parent in case halve failed */
        return node_parent(tn);
}

static void node_pull_suffix(struct key_vector *tn, unsigned char slen)
{
        unsigned char node_slen = tn->slen;

        while ((node_slen > tn->pos) && (node_slen > slen)) {
                slen = update_suffix(tn);
                if (node_slen == slen)
                        break;

                tn = node_parent(tn);
                node_slen = tn->slen;
        }
}

static void node_push_suffix(struct key_vector *tn, unsigned char slen)
{
        while (tn->slen < slen) {
                tn->slen = slen;
                tn = node_parent(tn);
        }
}

/* rcu_read_lock needs to be hold by caller from readside */
static struct key_vector *fib_find_node(struct trie *t,
                                        struct key_vector **tp, u32 key)
{
        struct key_vector *pn, *n = t->kv;
        unsigned long index = 0;

        do {
                pn = n;
                n = get_child_rcu(n, index);

                if (!n)
                        break;

                index = get_cindex(key, n);

                /* This bit of code is a bit tricky but it combines multiple
                 * checks into a single check.  The prefix consists of the
                 * prefix plus zeros for the bits in the cindex. The index
                 * is the difference between the key and this value.  From
                 * this we can actually derive several pieces of data.
                 *   if (index >= (1ul << bits))
                 *     we have a mismatch in skip bits and failed
                 *   else
                 *     we know the value is cindex
                 *
                 * This check is safe even if bits == KEYLENGTH due to the
                 * fact that we can only allocate a node with 32 bits if a
                 * long is greater than 32 bits.
                 */
                if (index >= (1ul << n->bits)) {
                        n = NULL;
                        break;
                }

                /* keep searching until we find a perfect match leaf or NULL */
        } while (IS_TNODE(n));

        *tp = pn;

        return n;
}

/* Return the first fib alias matching DSCP with
 * priority less than or equal to PRIO.
 * If 'find_first' is set, return the first matching
 * fib alias, regardless of DSCP and priority.
 */
static struct fib_alias *fib_find_alias(struct hlist_head *fah, u8 slen,
                                        dscp_t dscp, u32 prio, u32 tb_id,
                                        bool find_first)
{
        struct fib_alias *fa;

        if (!fah)
                return NULL;

        hlist_for_each_entry(fa, fah, fa_list) {
                /* Avoid Sparse warning when using dscp_t in inequalities */
                u8 __fa_dscp = inet_dscp_to_dsfield(fa->fa_dscp);
                u8 __dscp = inet_dscp_to_dsfield(dscp);

                if (fa->fa_slen < slen)
                        continue;
                if (fa->fa_slen != slen)
                        break;
                if (fa->tb_id > tb_id)
                        continue;
                if (fa->tb_id != tb_id)
                        break;
                if (find_first)
                        return fa;
                if (__fa_dscp > __dscp)
                        continue;
                if (fa->fa_info->fib_priority >= prio || __fa_dscp < __dscp)
                        return fa;
        }

        return NULL;
}

static struct fib_alias *
fib_find_matching_alias(struct net *net, const struct fib_rt_info *fri)
{
        u8 slen = KEYLENGTH - fri->dst_len;
        struct key_vector *l, *tp;
        struct fib_table *tb;
        struct fib_alias *fa;
        struct trie *t;

        tb = fib_get_table(net, fri->tb_id);
        if (!tb)
                return NULL;

        t = (struct trie *)tb->tb_data;
        l = fib_find_node(t, &tp, be32_to_cpu(fri->dst));
        if (!l)
                return NULL;

        hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
                if (fa->fa_slen == slen && fa->tb_id == fri->tb_id &&
                    fa->fa_dscp == fri->dscp && fa->fa_info == fri->fi &&
                    fa->fa_type == fri->type)
                        return fa;
        }

        return NULL;
}

void fib_alias_hw_flags_set(struct net *net, const struct fib_rt_info *fri)
{
        u8 fib_notify_on_flag_change;
        struct fib_alias *fa_match;
        struct sk_buff *skb;
        int err;

        rcu_read_lock();

        fa_match = fib_find_matching_alias(net, fri);
        if (!fa_match)
                goto out;

        /* These are paired with the WRITE_ONCE() happening in this function.
         * The reason is that we are only protected by RCU at this point.
         */
        if (READ_ONCE(fa_match->offload) == fri->offload &&
            READ_ONCE(fa_match->trap) == fri->trap &&
            READ_ONCE(fa_match->offload_failed) == fri->offload_failed)
                goto out;

        WRITE_ONCE(fa_match->offload, fri->offload);
        WRITE_ONCE(fa_match->trap, fri->trap);

        fib_notify_on_flag_change = READ_ONCE(net->ipv4.sysctl_fib_notify_on_flag_change);

        /* 2 means send notifications only if offload_failed was changed. */
        if (fib_notify_on_flag_change == 2 &&
            READ_ONCE(fa_match->offload_failed) == fri->offload_failed)
                goto out;

        WRITE_ONCE(fa_match->offload_failed, fri->offload_failed);

        if (!fib_notify_on_flag_change)
                goto out;

        skb = nlmsg_new(fib_nlmsg_size(fa_match->fa_info), GFP_ATOMIC);
        if (!skb) {
                err = -ENOBUFS;
                goto errout;
        }

        err = fib_dump_info(skb, 0, 0, RTM_NEWROUTE, fri, 0);
        if (err < 0) {
                /* -EMSGSIZE implies BUG in fib_nlmsg_size() */
                WARN_ON(err == -EMSGSIZE);
                kfree_skb(skb);
                goto errout;
        }

        rtnl_notify(skb, net, 0, RTNLGRP_IPV4_ROUTE, NULL, GFP_ATOMIC);
        goto out;

errout:
        rtnl_set_sk_err(net, RTNLGRP_IPV4_ROUTE, err);
out:
        rcu_read_unlock();
}
EXPORT_SYMBOL_GPL(fib_alias_hw_flags_set);

static void trie_rebalance(struct trie *t, struct key_vector *tn)
{
        while (!IS_TRIE(tn))
                tn = resize(t, tn);
}

static int fib_insert_node(struct trie *t, struct key_vector *tp,
                           struct fib_alias *new, t_key key)
{
        struct key_vector *n, *l;

        l = leaf_new(key, new);
        if (!l)
                goto noleaf;

        /* retrieve child from parent node */
        n = get_child(tp, get_index(key, tp));

        /* Case 2: n is a LEAF or a TNODE and the key doesn't match.
         *
         *  Add a new tnode here
         *  first tnode need some special handling
         *  leaves us in position for handling as case 3
         */
        if (n) {
                struct key_vector *tn;

                tn = tnode_new(key, __fls(key ^ n->key), 1);
                if (!tn)
                        goto notnode;

                /* initialize routes out of node */
                NODE_INIT_PARENT(tn, tp);
                put_child(tn, get_index(key, tn) ^ 1, n);

                /* start adding routes into the node */
                put_child_root(tp, key, tn);
                node_set_parent(n, tn);

                /* parent now has a NULL spot where the leaf can go */
                tp = tn;
        }

        /* Case 3: n is NULL, and will just insert a new leaf */
        node_push_suffix(tp, new->fa_slen);
        NODE_INIT_PARENT(l, tp);
        put_child_root(tp, key, l);
        trie_rebalance(t, tp);

        return 0;
notnode:
        node_free(l);
noleaf:
        return -ENOMEM;
}

static int fib_insert_alias(struct trie *t, struct key_vector *tp,
                            struct key_vector *l, struct fib_alias *new,
                            struct fib_alias *fa, t_key key)
{
        if (!l)
                return fib_insert_node(t, tp, new, key);

        if (fa) {
                hlist_add_before_rcu(&new->fa_list, &fa->fa_list);
        } else {
                struct fib_alias *last;

                hlist_for_each_entry(last, &l->leaf, fa_list) {
                        if (new->fa_slen < last->fa_slen)
                                break;
                        if ((new->fa_slen == last->fa_slen) &&
                            (new->tb_id > last->tb_id))
                                break;
                        fa = last;
                }

                if (fa)
                        hlist_add_behind_rcu(&new->fa_list, &fa->fa_list);
                else
                        hlist_add_head_rcu(&new->fa_list, &l->leaf);
        }

        /* if we added to the tail node then we need to update slen */
        if (l->slen < new->fa_slen) {
                l->slen = new->fa_slen;
                node_push_suffix(tp, new->fa_slen);
        }

        return 0;
}

static void fib_remove_alias(struct trie *t, struct key_vector *tp,
                             struct key_vector *l, struct fib_alias *old);

/* Caller must hold RTNL. */
int fib_table_insert(struct net *net, struct fib_table *tb,
                     struct fib_config *cfg, struct netlink_ext_ack *extack)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct fib_alias *fa, *new_fa;
        struct key_vector *l, *tp;
        u16 nlflags = NLM_F_EXCL;
        struct fib_info *fi;
        u8 plen = cfg->fc_dst_len;
        u8 slen = KEYLENGTH - plen;
        dscp_t dscp;
        u32 key;
        int err;

        key = ntohl(cfg->fc_dst);

        pr_debug("Insert table=%u %08x/%d\n", tb->tb_id, key, plen);

        fi = fib_create_info(cfg, extack);
        if (IS_ERR(fi)) {
                err = PTR_ERR(fi);
                goto err;
        }

        dscp = cfg->fc_dscp;
        l = fib_find_node(t, &tp, key);
        fa = l ? fib_find_alias(&l->leaf, slen, dscp, fi->fib_priority,
                                tb->tb_id, false) : NULL;

        /* Now fa, if non-NULL, points to the first fib alias
         * with the same keys [prefix,dscp,priority], if such key already
         * exists or to the node before which we will insert new one.
         *
         * If fa is NULL, we will need to allocate a new one and
         * insert to the tail of the section matching the suffix length
         * of the new alias.
         */

        if (fa && fa->fa_dscp == dscp &&
            fa->fa_info->fib_priority == fi->fib_priority) {
                struct fib_alias *fa_first, *fa_match;

                err = -EEXIST;
                if (cfg->fc_nlflags & NLM_F_EXCL)
                        goto out;

                nlflags &= ~NLM_F_EXCL;

                /* We have 2 goals:
                 * 1. Find exact match for type, scope, fib_info to avoid
                 * duplicate routes
                 * 2. Find next 'fa' (or head), NLM_F_APPEND inserts before it
                 */
                fa_match = NULL;
                fa_first = fa;
                hlist_for_each_entry_from(fa, fa_list) {
                        if ((fa->fa_slen != slen) ||
                            (fa->tb_id != tb->tb_id) ||
                            (fa->fa_dscp != dscp))
                                break;
                        if (fa->fa_info->fib_priority != fi->fib_priority)
                                break;
                        if (fa->fa_type == cfg->fc_type &&
                            fa->fa_info == fi) {
                                fa_match = fa;
                                break;
                        }
                }

                if (cfg->fc_nlflags & NLM_F_REPLACE) {
                        struct fib_info *fi_drop;
                        u8 state;

                        nlflags |= NLM_F_REPLACE;
                        fa = fa_first;
                        if (fa_match) {
                                if (fa == fa_match)
                                        err = 0;
                                goto out;
                        }
                        err = -ENOBUFS;
                        new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
                        if (!new_fa)
                                goto out;

                        fi_drop = fa->fa_info;
                        new_fa->fa_dscp = fa->fa_dscp;
                        new_fa->fa_info = fi;
                        new_fa->fa_type = cfg->fc_type;
                        state = fa->fa_state;
                        new_fa->fa_state = state & ~FA_S_ACCESSED;
                        new_fa->fa_slen = fa->fa_slen;
                        new_fa->tb_id = tb->tb_id;
                        new_fa->fa_default = -1;
                        new_fa->offload = 0;
                        new_fa->trap = 0;
                        new_fa->offload_failed = 0;

                        hlist_replace_rcu(&fa->fa_list, &new_fa->fa_list);

                        if (fib_find_alias(&l->leaf, fa->fa_slen, 0, 0,
                                           tb->tb_id, true) == new_fa) {
                                enum fib_event_type fib_event;

                                fib_event = FIB_EVENT_ENTRY_REPLACE;
                                err = call_fib_entry_notifiers(net, fib_event,
                                                               key, plen,
                                                               new_fa, extack);
                                if (err) {
                                        hlist_replace_rcu(&new_fa->fa_list,
                                                          &fa->fa_list);
                                        goto out_free_new_fa;
                                }
                        }

                        rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen,
                                  tb->tb_id, &cfg->fc_nlinfo, nlflags);

                        alias_free_mem_rcu(fa);

                        fib_release_info(fi_drop);
                        if (state & FA_S_ACCESSED)
                                rt_cache_flush(cfg->fc_nlinfo.nl_net);

                        goto succeeded;
                }
                /* Error if we find a perfect match which
                 * uses the same scope, type, and nexthop
                 * information.
                 */
                if (fa_match)
                        goto out;

                if (cfg->fc_nlflags & NLM_F_APPEND)
                        nlflags |= NLM_F_APPEND;
                else
                        fa = fa_first;
        }
        err = -ENOENT;
        if (!(cfg->fc_nlflags & NLM_F_CREATE))
                goto out;

        nlflags |= NLM_F_CREATE;
        err = -ENOBUFS;
        new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
        if (!new_fa)
                goto out;

        new_fa->fa_info = fi;
        new_fa->fa_dscp = dscp;
        new_fa->fa_type = cfg->fc_type;
        new_fa->fa_state = 0;
        new_fa->fa_slen = slen;
        new_fa->tb_id = tb->tb_id;
        new_fa->fa_default = -1;
        new_fa->offload = 0;
        new_fa->trap = 0;
        new_fa->offload_failed = 0;

        /* Insert new entry to the list. */
        err = fib_insert_alias(t, tp, l, new_fa, fa, key);
        if (err)
                goto out_free_new_fa;

        /* The alias was already inserted, so the node must exist. */
        l = l ? l : fib_find_node(t, &tp, key);
        if (WARN_ON_ONCE(!l)) {
                err = -ENOENT;
                goto out_free_new_fa;
        }

        if (fib_find_alias(&l->leaf, new_fa->fa_slen, 0, 0, tb->tb_id, true) ==
            new_fa) {
                enum fib_event_type fib_event;

                fib_event = FIB_EVENT_ENTRY_REPLACE;
                err = call_fib_entry_notifiers(net, fib_event, key, plen,
                                               new_fa, extack);
                if (err)
                        goto out_remove_new_fa;
        }

        if (!plen)
                tb->tb_num_default++;

        rt_cache_flush(cfg->fc_nlinfo.nl_net);
        rtmsg_fib(RTM_NEWROUTE, htonl(key), new_fa, plen, new_fa->tb_id,
                  &cfg->fc_nlinfo, nlflags);
succeeded:
        return 0;

out_remove_new_fa:
        fib_remove_alias(t, tp, l, new_fa);
out_free_new_fa:
        kmem_cache_free(fn_alias_kmem, new_fa);
out:
        fib_release_info(fi);
err:
        return err;
}

static inline t_key prefix_mismatch(t_key key, struct key_vector *n)
{
        t_key prefix = n->key;

        return (key ^ prefix) & (prefix | -prefix);
}

bool fib_lookup_good_nhc(const struct fib_nh_common *nhc, int fib_flags,
                         const struct flowi4 *flp)
{
        if (nhc->nhc_flags & RTNH_F_DEAD)
                return false;

        if (ip_ignore_linkdown(nhc->nhc_dev) &&
            nhc->nhc_flags & RTNH_F_LINKDOWN &&
            !(fib_flags & FIB_LOOKUP_IGNORE_LINKSTATE))
                return false;

        if (flp->flowi4_oif && flp->flowi4_oif != nhc->nhc_oif)
                return false;

        return true;
}

/* should be called with rcu_read_lock */
int fib_table_lookup(struct fib_table *tb, const struct flowi4 *flp,
                     struct fib_result *res, int fib_flags)
{
        struct trie *t = (struct trie *) tb->tb_data;
#ifdef CONFIG_IP_FIB_TRIE_STATS
        struct trie_use_stats __percpu *stats = t->stats;
#endif
        const t_key key = ntohl(flp->daddr);
        struct key_vector *n, *pn;
        struct fib_alias *fa;
        unsigned long index;
        t_key cindex;

        pn = t->kv;
        cindex = 0;

        n = get_child_rcu(pn, cindex);
        if (!n) {
                trace_fib_table_lookup(tb->tb_id, flp, NULL, -EAGAIN);
                return -EAGAIN;
        }

#ifdef CONFIG_IP_FIB_TRIE_STATS
        this_cpu_inc(stats->gets);
#endif

        /* Step 1: Travel to the longest prefix match in the trie */
        for (;;) {
                index = get_cindex(key, n);

                /* This bit of code is a bit tricky but it combines multiple
                 * checks into a single check.  The prefix consists of the
                 * prefix plus zeros for the "bits" in the prefix. The index
                 * is the difference between the key and this value.  From
                 * this we can actually derive several pieces of data.
                 *   if (index >= (1ul << bits))
                 *     we have a mismatch in skip bits and failed
                 *   else
                 *     we know the value is cindex
                 *
                 * This check is safe even if bits == KEYLENGTH due to the
                 * fact that we can only allocate a node with 32 bits if a
                 * long is greater than 32 bits.
                 */
                if (index >= (1ul << n->bits))
                        break;

                /* we have found a leaf. Prefixes have already been compared */
                if (IS_LEAF(n))
                        goto found;

                /* only record pn and cindex if we are going to be chopping
                 * bits later.  Otherwise we are just wasting cycles.
                 */
                if (n->slen > n->pos) {
                        pn = n;
                        cindex = index;
                }

                n = get_child_rcu(n, index);
                if (unlikely(!n))
                        goto backtrace;
        }

        /* Step 2: Sort out leaves and begin backtracing for longest prefix */
        for (;;) {
                /* record the pointer where our next node pointer is stored */
                struct key_vector __rcu **cptr = n->tnode;

                /* This test verifies that none of the bits that differ
                 * between the key and the prefix exist in the region of
                 * the lsb and higher in the prefix.
                 */
                if (unlikely(prefix_mismatch(key, n)) || (n->slen == n->pos))
                        goto backtrace;

                /* exit out and process leaf */
                if (unlikely(IS_LEAF(n)))
                        break;

                /* Don't bother recording parent info.  Since we are in
                 * prefix match mode we will have to come back to wherever
                 * we started this traversal anyway
                 */

                while ((n = rcu_dereference(*cptr)) == NULL) {
backtrace:
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        if (!n)
                                this_cpu_inc(stats->null_node_hit);
#endif
                        /* If we are at cindex 0 there are no more bits for
                         * us to strip at this level so we must ascend back
                         * up one level to see if there are any more bits to
                         * be stripped there.
                         */
                        while (!cindex) {
                                t_key pkey = pn->key;

                                /* If we don't have a parent then there is
                                 * nothing for us to do as we do not have any
                                 * further nodes to parse.
                                 */
                                if (IS_TRIE(pn)) {
                                        trace_fib_table_lookup(tb->tb_id, flp,
                                                               NULL, -EAGAIN);
                                        return -EAGAIN;
                                }
#ifdef CONFIG_IP_FIB_TRIE_STATS
                                this_cpu_inc(stats->backtrack);
#endif
                                /* Get Child's index */
                                pn = node_parent_rcu(pn);
                                cindex = get_index(pkey, pn);
                        }

                        /* strip the least significant bit from the cindex */
                        cindex &= cindex - 1;

                        /* grab pointer for next child node */
                        cptr = &pn->tnode[cindex];
                }
        }

found:
        /* this line carries forward the xor from earlier in the function */
        index = key ^ n->key;

        /* Step 3: Process the leaf, if that fails fall back to backtracing */
        hlist_for_each_entry_rcu(fa, &n->leaf, fa_list) {
                struct fib_info *fi = fa->fa_info;
                struct fib_nh_common *nhc;
                int nhsel, err;

                if ((BITS_PER_LONG > KEYLENGTH) || (fa->fa_slen < KEYLENGTH)) {
                        if (index >= (1ul << fa->fa_slen))
                                continue;
                }
                if (fa->fa_dscp && !fib_dscp_masked_match(fa->fa_dscp, flp))
                        continue;
                /* Paired with WRITE_ONCE() in fib_release_info() */
                if (READ_ONCE(fi->fib_dead))
                        continue;
                if (fa->fa_info->fib_scope < flp->flowi4_scope)
                        continue;
                fib_alias_accessed(fa);
                err = fib_props[fa->fa_type].error;
                if (unlikely(err < 0)) {
out_reject:
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        this_cpu_inc(stats->semantic_match_passed);
#endif
                        trace_fib_table_lookup(tb->tb_id, flp, NULL, err);
                        return err;
                }
                if (fi->fib_flags & RTNH_F_DEAD)
                        continue;

                if (unlikely(fi->nh)) {
                        if (nexthop_is_blackhole(fi->nh)) {
                                err = fib_props[RTN_BLACKHOLE].error;
                                goto out_reject;
                        }

                        nhc = nexthop_get_nhc_lookup(fi->nh, fib_flags, flp,
                                                     &nhsel);
                        if (nhc)
                                goto set_result;
                        goto miss;
                }

                for (nhsel = 0; nhsel < fib_info_num_path(fi); nhsel++) {
                        nhc = fib_info_nhc(fi, nhsel);

                        if (!fib_lookup_good_nhc(nhc, fib_flags, flp))
                                continue;
set_result:
                        if (!(fib_flags & FIB_LOOKUP_NOREF))
                                refcount_inc(&fi->fib_clntref);

                        res->prefix = htonl(n->key);
                        res->prefixlen = KEYLENGTH - fa->fa_slen;
                        res->nh_sel = nhsel;
                        res->nhc = nhc;
                        res->type = fa->fa_type;
                        res->scope = fi->fib_scope;
                        res->dscp = fa->fa_dscp;
                        res->fi = fi;
                        res->table = tb;
                        res->fa_head = &n->leaf;
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        this_cpu_inc(stats->semantic_match_passed);
#endif
                        trace_fib_table_lookup(tb->tb_id, flp, nhc, err);

                        return err;
                }
        }
miss:
#ifdef CONFIG_IP_FIB_TRIE_STATS
        this_cpu_inc(stats->semantic_match_miss);
#endif
        goto backtrace;
}
EXPORT_SYMBOL_GPL(fib_table_lookup);

static void fib_remove_alias(struct trie *t, struct key_vector *tp,
                             struct key_vector *l, struct fib_alias *old)
{
        /* record the location of the previous list_info entry */
        struct hlist_node **pprev = old->fa_list.pprev;
        struct fib_alias *fa = hlist_entry(pprev, typeof(*fa), fa_list.next);

        /* remove the fib_alias from the list */
        hlist_del_rcu(&old->fa_list);

        /* if we emptied the list this leaf will be freed and we can sort
         * out parent suffix lengths as a part of trie_rebalance
         */
        if (hlist_empty(&l->leaf)) {
                if (tp->slen == l->slen)
                        node_pull_suffix(tp, tp->pos);
                put_child_root(tp, l->key, NULL);
                node_free(l);
                trie_rebalance(t, tp);
                return;
        }

        /* only access fa if it is pointing at the last valid hlist_node */
        if (*pprev)
                return;

        /* update the trie with the latest suffix length */
        l->slen = fa->fa_slen;
        node_pull_suffix(tp, fa->fa_slen);
}

static void fib_notify_alias_delete(struct net *net, u32 key,
                                    struct hlist_head *fah,
                                    struct fib_alias *fa_to_delete,
                                    struct netlink_ext_ack *extack)
{
        struct fib_alias *fa_next, *fa_to_notify;
        u32 tb_id = fa_to_delete->tb_id;
        u8 slen = fa_to_delete->fa_slen;
        enum fib_event_type fib_event;

        /* Do not notify if we do not care about the route. */
        if (fib_find_alias(fah, slen, 0, 0, tb_id, true) != fa_to_delete)
                return;

        /* Determine if the route should be replaced by the next route in the
         * list.
         */
        fa_next = hlist_entry_safe(fa_to_delete->fa_list.next,
                                   struct fib_alias, fa_list);
        if (fa_next && fa_next->fa_slen == slen && fa_next->tb_id == tb_id) {
                fib_event = FIB_EVENT_ENTRY_REPLACE;
                fa_to_notify = fa_next;
        } else {
                fib_event = FIB_EVENT_ENTRY_DEL;
                fa_to_notify = fa_to_delete;
        }
        call_fib_entry_notifiers(net, fib_event, key, KEYLENGTH - slen,
                                 fa_to_notify, extack);
}

/* Caller must hold RTNL. */
int fib_table_delete(struct net *net, struct fib_table *tb,
                     struct fib_config *cfg, struct netlink_ext_ack *extack)
{
        struct trie *t = (struct trie *) tb->tb_data;
        struct fib_alias *fa, *fa_to_delete;
        struct key_vector *l, *tp;
        u8 plen = cfg->fc_dst_len;
        u8 slen = KEYLENGTH - plen;
        dscp_t dscp;
        u32 key;

        key = ntohl(cfg->fc_dst);

        l = fib_find_node(t, &tp, key);
        if (!l)
                return -ESRCH;

        dscp = cfg->fc_dscp;
        fa = fib_find_alias(&l->leaf, slen, dscp, 0, tb->tb_id, false);
        if (!fa)
                return -ESRCH;

        pr_debug("Deleting %08x/%d dsfield=0x%02x t=%p\n", key, plen,
                 inet_dscp_to_dsfield(dscp), t);

        fa_to_delete = NULL;
        hlist_for_each_entry_from(fa, fa_list) {
                struct fib_info *fi = fa->fa_info;

                if ((fa->fa_slen != slen) ||
                    (fa->tb_id != tb->tb_id) ||
                    (fa->fa_dscp != dscp))
                        break;

                if ((!cfg->fc_type || fa->fa_type == cfg->fc_type) &&
                    (cfg->fc_scope == RT_SCOPE_NOWHERE ||
                     fa->fa_info->fib_scope == cfg->fc_scope) &&
                    (!cfg->fc_prefsrc ||
                     fi->fib_prefsrc == cfg->fc_prefsrc) &&
                    (!cfg->fc_protocol ||
                     fi->fib_protocol == cfg->fc_protocol) &&
                    fib_nh_match(net, cfg, fi, extack) == 0 &&
                    fib_metrics_match(cfg, fi)) {
                        fa_to_delete = fa;
                        break;
                }
        }

        if (!fa_to_delete)
                return -ESRCH;

        fib_notify_alias_delete(net, key, &l->leaf, fa_to_delete, extack);
        rtmsg_fib(RTM_DELROUTE, htonl(key), fa_to_delete, plen, tb->tb_id,
                  &cfg->fc_nlinfo, 0);

        if (!plen)
                tb->tb_num_default--;

        fib_remove_alias(t, tp, l, fa_to_delete);

        if (fa_to_delete->fa_state & FA_S_ACCESSED)
                rt_cache_flush(cfg->fc_nlinfo.nl_net);

        fib_release_info(fa_to_delete->fa_info);
        alias_free_mem_rcu(fa_to_delete);
        return 0;
}

/* Scan for the next leaf starting at the provided key value */
static struct key_vector *leaf_walk_rcu(struct key_vector **tn, t_key key)
{
        struct key_vector *pn, *n = *tn;
        unsigned long cindex;

        /* this loop is meant to try and find the key in the trie */
        do {
                /* record parent and next child index */
                pn = n;
                cindex = (key > pn->key) ? get_index(key, pn) : 0;

                if (cindex >> pn->bits)
                        break;

                /* descend into the next child */
                n = get_child_rcu(pn, cindex++);
                if (!n)
                        break;

                /* guarantee forward progress on the keys */
                if (IS_LEAF(n) && (n->key >= key))
                        goto found;
        } while (IS_TNODE(n));

        /* this loop will search for the next leaf with a greater key */
        while (!IS_TRIE(pn)) {
                /* if we exhausted the parent node we will need to climb */
                if (cindex >= (1ul << pn->bits)) {
                        t_key pkey = pn->key;

                        pn = node_parent_rcu(pn);
                        cindex = get_index(pkey, pn) + 1;
                        continue;
                }

                /* grab the next available node */
                n = get_child_rcu(pn, cindex++);
                if (!n)
                        continue;

                /* no need to compare keys since we bumped the index */
                if (IS_LEAF(n))
                        goto found;

                /* Rescan start scanning in new node */
                pn = n;
                cindex = 0;
        }

        *tn = pn;
        return NULL; /* Root of trie */
found:
        /* if we are at the limit for keys just return NULL for the tnode */
        *tn = pn;
        return n;
}

static void fib_trie_free(struct fib_table *tb)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct key_vector *pn = t->kv;
        unsigned long cindex = 1;
        struct hlist_node *tmp;
        struct fib_alias *fa;

        /* walk trie in reverse order and free everything */
        for (;;) {
                struct key_vector *n;

                if (!(cindex--)) {
                        t_key pkey = pn->key;

                        if (IS_TRIE(pn))
                                break;

                        n = pn;
                        pn = node_parent(pn);

                        /* drop emptied tnode */
                        put_child_root(pn, n->key, NULL);
                        node_free(n);

                        cindex = get_index(pkey, pn);

                        continue;
                }

                /* grab the next available node */
                n = get_child(pn, cindex);
                if (!n)
                        continue;

                if (IS_TNODE(n)) {
                        /* record pn and cindex for leaf walking */
                        pn = n;
                        cindex = 1ul << n->bits;

                        continue;
                }

                hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
                        hlist_del_rcu(&fa->fa_list);
                        alias_free_mem_rcu(fa);
                }

                put_child_root(pn, n->key, NULL);
                node_free(n);
        }

#ifdef CONFIG_IP_FIB_TRIE_STATS
        free_percpu(t->stats);
#endif
        kfree(tb);
}

struct fib_table *fib_trie_unmerge(struct fib_table *oldtb)
{
        struct trie *ot = (struct trie *)oldtb->tb_data;
        struct key_vector *l, *tp = ot->kv;
        struct fib_table *local_tb;
        struct fib_alias *fa;
        struct trie *lt;
        t_key key = 0;

        if (oldtb->tb_data == oldtb->__data)
                return oldtb;

        local_tb = fib_trie_table(RT_TABLE_LOCAL, NULL);
        if (!local_tb)
                return NULL;

        lt = (struct trie *)local_tb->tb_data;

        while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
                struct key_vector *local_l = NULL, *local_tp;

                hlist_for_each_entry(fa, &l->leaf, fa_list) {
                        struct fib_alias *new_fa;

                        if (local_tb->tb_id != fa->tb_id)
                                continue;

                        /* clone fa for new local table */
                        new_fa = kmem_cache_alloc(fn_alias_kmem, GFP_KERNEL);
                        if (!new_fa)
                                goto out;

                        memcpy(new_fa, fa, sizeof(*fa));

                        /* insert clone into table */
                        if (!local_l)
                                local_l = fib_find_node(lt, &local_tp, l->key);

                        if (fib_insert_alias(lt, local_tp, local_l, new_fa,
                                             NULL, l->key)) {
                                kmem_cache_free(fn_alias_kmem, new_fa);
                                goto out;
                        }
                }

                /* stop loop if key wrapped back to 0 */
                key = l->key + 1;
                if (key < l->key)
                        break;
        }

        return local_tb;
out:
        fib_trie_free(local_tb);

        return NULL;
}

/* Caller must hold RTNL */
void fib_table_flush_external(struct fib_table *tb)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct key_vector *pn = t->kv;
        unsigned long cindex = 1;
        struct hlist_node *tmp;
        struct fib_alias *fa;

        /* walk trie in reverse order */
        for (;;) {
                unsigned char slen = 0;
                struct key_vector *n;

                if (!(cindex--)) {
                        t_key pkey = pn->key;

                        /* cannot resize the trie vector */
                        if (IS_TRIE(pn))
                                break;

                        /* update the suffix to address pulled leaves */
                        if (pn->slen > pn->pos)
                                update_suffix(pn);

                        /* resize completed node */
                        pn = resize(t, pn);
                        cindex = get_index(pkey, pn);

                        continue;
                }

                /* grab the next available node */
                n = get_child(pn, cindex);
                if (!n)
                        continue;

                if (IS_TNODE(n)) {
                        /* record pn and cindex for leaf walking */
                        pn = n;
                        cindex = 1ul << n->bits;

                        continue;
                }

                hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
                        /* if alias was cloned to local then we just
                         * need to remove the local copy from main
                         */
                        if (tb->tb_id != fa->tb_id) {
                                hlist_del_rcu(&fa->fa_list);
                                alias_free_mem_rcu(fa);
                                continue;
                        }

                        /* record local slen */
                        slen = fa->fa_slen;
                }

                /* update leaf slen */
                n->slen = slen;

                if (hlist_empty(&n->leaf)) {
                        put_child_root(pn, n->key, NULL);
                        node_free(n);
                }
        }
}

/* Caller must hold RTNL. */
int fib_table_flush(struct net *net, struct fib_table *tb, bool flush_all)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct nl_info info = { .nl_net = net };
        struct key_vector *pn = t->kv;
        unsigned long cindex = 1;
        struct hlist_node *tmp;
        struct fib_alias *fa;
        int found = 0;

        /* walk trie in reverse order */
        for (;;) {
                unsigned char slen = 0;
                struct key_vector *n;

                if (!(cindex--)) {
                        t_key pkey = pn->key;

                        /* cannot resize the trie vector */
                        if (IS_TRIE(pn))
                                break;

                        /* update the suffix to address pulled leaves */
                        if (pn->slen > pn->pos)
                                update_suffix(pn);

                        /* resize completed node */
                        pn = resize(t, pn);
                        cindex = get_index(pkey, pn);

                        continue;
                }

                /* grab the next available node */
                n = get_child(pn, cindex);
                if (!n)
                        continue;

                if (IS_TNODE(n)) {
                        /* record pn and cindex for leaf walking */
                        pn = n;
                        cindex = 1ul << n->bits;

                        continue;
                }

                hlist_for_each_entry_safe(fa, tmp, &n->leaf, fa_list) {
                        struct fib_info *fi = fa->fa_info;

                        if (!fi || tb->tb_id != fa->tb_id ||
                            (!(fi->fib_flags & RTNH_F_DEAD) &&
                             !fib_props[fa->fa_type].error)) {
                                slen = fa->fa_slen;
                                continue;
                        }

                        /* Do not flush error routes if network namespace is
                         * not being dismantled
                         */
                        if (!flush_all && fib_props[fa->fa_type].error) {
                                slen = fa->fa_slen;
                                continue;
                        }

                        fib_notify_alias_delete(net, n->key, &n->leaf, fa,
                                                NULL);
                        if (fi->pfsrc_removed)
                                rtmsg_fib(RTM_DELROUTE, htonl(n->key), fa,
                                          KEYLENGTH - fa->fa_slen, tb->tb_id, &info, 0);
                        hlist_del_rcu(&fa->fa_list);
                        fib_release_info(fa->fa_info);
                        alias_free_mem_rcu(fa);
                        found++;
                }

                /* update leaf slen */
                n->slen = slen;

                if (hlist_empty(&n->leaf)) {
                        put_child_root(pn, n->key, NULL);
                        node_free(n);
                }
        }

        pr_debug("trie_flush found=%d\n", found);
        return found;
}

/* derived from fib_trie_free */
static void __fib_info_notify_update(struct net *net, struct fib_table *tb,
                                     struct nl_info *info)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct key_vector *pn = t->kv;
        unsigned long cindex = 1;
        struct fib_alias *fa;

        for (;;) {
                struct key_vector *n;

                if (!(cindex--)) {
                        t_key pkey = pn->key;

                        if (IS_TRIE(pn))
                                break;

                        pn = node_parent(pn);
                        cindex = get_index(pkey, pn);
                        continue;
                }

                /* grab the next available node */
                n = get_child(pn, cindex);
                if (!n)
                        continue;

                if (IS_TNODE(n)) {
                        /* record pn and cindex for leaf walking */
                        pn = n;
                        cindex = 1ul << n->bits;

                        continue;
                }

                hlist_for_each_entry(fa, &n->leaf, fa_list) {
                        struct fib_info *fi = fa->fa_info;

                        if (!fi || !fi->nh_updated || fa->tb_id != tb->tb_id)
                                continue;

                        rtmsg_fib(RTM_NEWROUTE, htonl(n->key), fa,
                                  KEYLENGTH - fa->fa_slen, tb->tb_id,
                                  info, NLM_F_REPLACE);
                }
        }
}

void fib_info_notify_update(struct net *net, struct nl_info *info)
{
        unsigned int h;

        for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
                struct hlist_head *head = &net->ipv4.fib_table_hash[h];
                struct fib_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb_hlist,
                                         lockdep_rtnl_is_held())
                        __fib_info_notify_update(net, tb, info);
        }
}

static int fib_leaf_notify(struct key_vector *l, struct fib_table *tb,
                           struct notifier_block *nb,
                           struct netlink_ext_ack *extack)
{
        struct fib_alias *fa;
        int last_slen = -1;
        int err;

        hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
                struct fib_info *fi = fa->fa_info;

                if (!fi)
                        continue;

                /* local and main table can share the same trie,
                 * so don't notify twice for the same entry.
                 */
                if (tb->tb_id != fa->tb_id)
                        continue;

                if (fa->fa_slen == last_slen)
                        continue;

                last_slen = fa->fa_slen;
                err = call_fib_entry_notifier(nb, FIB_EVENT_ENTRY_REPLACE,
                                              l->key, KEYLENGTH - fa->fa_slen,
                                              fa, extack);
                if (err)
                        return err;
        }
        return 0;
}

static int fib_table_notify(struct fib_table *tb, struct notifier_block *nb,
                            struct netlink_ext_ack *extack)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct key_vector *l, *tp = t->kv;
        t_key key = 0;
        int err;

        while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
                err = fib_leaf_notify(l, tb, nb, extack);
                if (err)
                        return err;

                key = l->key + 1;
                /* stop in case of wrap around */
                if (key < l->key)
                        break;
        }
        return 0;
}

int fib_notify(struct net *net, struct notifier_block *nb,
               struct netlink_ext_ack *extack)
{
        unsigned int h;
        int err;

        for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
                struct hlist_head *head = &net->ipv4.fib_table_hash[h];
                struct fib_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb_hlist) {
                        err = fib_table_notify(tb, nb, extack);
                        if (err)
                                return err;
                }
        }
        return 0;
}

static void __trie_free_rcu(struct rcu_head *head)
{
        struct fib_table *tb = container_of(head, struct fib_table, rcu);
#ifdef CONFIG_IP_FIB_TRIE_STATS
        struct trie *t = (struct trie *)tb->tb_data;

        if (tb->tb_data == tb->__data)
                free_percpu(t->stats);
#endif /* CONFIG_IP_FIB_TRIE_STATS */
        kfree(tb);
}

void fib_free_table(struct fib_table *tb)
{
        call_rcu(&tb->rcu, __trie_free_rcu);
}

static int fn_trie_dump_leaf(struct key_vector *l, struct fib_table *tb,
                             struct sk_buff *skb, struct netlink_callback *cb,
                             struct fib_dump_filter *filter)
{
        unsigned int flags = NLM_F_MULTI;
        __be32 xkey = htonl(l->key);
        int i, s_i, i_fa, s_fa, err;
        struct fib_alias *fa;

        if (filter->filter_set ||
            !filter->dump_exceptions || !filter->dump_routes)
                flags |= NLM_F_DUMP_FILTERED;

        s_i = cb->args[4];
        s_fa = cb->args[5];
        i = 0;

        /* rcu_read_lock is hold by caller */
        hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
                struct fib_info *fi = fa->fa_info;

                if (i < s_i)
                        goto next;

                i_fa = 0;

                if (tb->tb_id != fa->tb_id)
                        goto next;

                if (filter->filter_set) {
                        if (filter->rt_type && fa->fa_type != filter->rt_type)
                                goto next;

                        if ((filter->protocol &&
                             fi->fib_protocol != filter->protocol))
                                goto next;

                        if (filter->dev &&
                            !fib_info_nh_uses_dev(fi, filter->dev))
                                goto next;
                }

                if (filter->dump_routes) {
                        if (!s_fa) {
                                struct fib_rt_info fri;

                                fri.fi = fi;
                                fri.tb_id = tb->tb_id;
                                fri.dst = xkey;
                                fri.dst_len = KEYLENGTH - fa->fa_slen;
                                fri.dscp = fa->fa_dscp;
                                fri.type = fa->fa_type;
                                fri.offload = READ_ONCE(fa->offload);
                                fri.trap = READ_ONCE(fa->trap);
                                fri.offload_failed = READ_ONCE(fa->offload_failed);
                                err = fib_dump_info(skb,
                                                    NETLINK_CB(cb->skb).portid,
                                                    cb->nlh->nlmsg_seq,
                                                    RTM_NEWROUTE, &fri, flags);
                                if (err < 0)
                                        goto stop;
                        }

                        i_fa++;
                }

                if (filter->dump_exceptions) {
                        err = fib_dump_info_fnhe(skb, cb, tb->tb_id, fi,
                                                 &i_fa, s_fa, flags);
                        if (err < 0)
                                goto stop;
                }

next:
                i++;
        }

        cb->args[4] = i;
        return skb->len;

stop:
        cb->args[4] = i;
        cb->args[5] = i_fa;
        return err;
}

/* rcu_read_lock needs to be hold by caller from readside */
int fib_table_dump(struct fib_table *tb, struct sk_buff *skb,
                   struct netlink_callback *cb, struct fib_dump_filter *filter)
{
        struct trie *t = (struct trie *)tb->tb_data;
        struct key_vector *l, *tp = t->kv;
        /* Dump starting at last key.
         * Note: 0.0.0.0/0 (ie default) is first key.
         */
        int count = cb->args[2];
        t_key key = cb->args[3];

        /* First time here, count and key are both always 0. Count > 0
         * and key == 0 means the dump has wrapped around and we are done.
         */
        if (count && !key)
                return 0;

        while ((l = leaf_walk_rcu(&tp, key)) != NULL) {
                int err;

                err = fn_trie_dump_leaf(l, tb, skb, cb, filter);
                if (err < 0) {
                        cb->args[3] = key;
                        cb->args[2] = count;
                        return err;
                }

                ++count;
                key = l->key + 1;

                memset(&cb->args[4], 0,
                       sizeof(cb->args) - 4*sizeof(cb->args[0]));

                /* stop loop if key wrapped back to 0 */
                if (key < l->key)
                        break;
        }

        cb->args[3] = key;
        cb->args[2] = count;

        return 0;
}

void __init fib_trie_init(void)
{
        fn_alias_kmem = kmem_cache_create("ip_fib_alias",
                                          sizeof(struct fib_alias),
                                          0, SLAB_PANIC | SLAB_ACCOUNT, NULL);

        trie_leaf_kmem = kmem_cache_create("ip_fib_trie",
                                           LEAF_SIZE,
                                           0, SLAB_PANIC | SLAB_ACCOUNT, NULL);
}

struct fib_table *fib_trie_table(u32 id, struct fib_table *alias)
{
        struct fib_table *tb;
        struct trie *t;
        size_t sz = sizeof(*tb);

        if (!alias)
                sz += sizeof(struct trie);

        tb = kzalloc(sz, GFP_KERNEL);
        if (!tb)
                return NULL;

        tb->tb_id = id;
        tb->tb_num_default = 0;
        tb->tb_data = (alias ? alias->__data : tb->__data);

        if (alias)
                return tb;

        t = (struct trie *) tb->tb_data;
        t->kv[0].pos = KEYLENGTH;
        t->kv[0].slen = KEYLENGTH;
#ifdef CONFIG_IP_FIB_TRIE_STATS
        t->stats = alloc_percpu(struct trie_use_stats);
        if (!t->stats) {
                kfree(tb);
                tb = NULL;
        }
#endif

        return tb;
}

#ifdef CONFIG_PROC_FS
/* Depth first Trie walk iterator */
struct fib_trie_iter {
        struct seq_net_private p;
        struct fib_table *tb;
        struct key_vector *tnode;
        unsigned int index;
        unsigned int depth;
};

static struct key_vector *fib_trie_get_next(struct fib_trie_iter *iter)
{
        unsigned long cindex = iter->index;
        struct key_vector *pn = iter->tnode;
        t_key pkey;

        pr_debug("get_next iter={node=%p index=%d depth=%d}\n",
                 iter->tnode, iter->index, iter->depth);

        while (!IS_TRIE(pn)) {
                while (cindex < child_length(pn)) {
                        struct key_vector *n = get_child_rcu(pn, cindex++);

                        if (!n)
                                continue;

                        if (IS_LEAF(n)) {
                                iter->tnode = pn;
                                iter->index = cindex;
                        } else {
                                /* push down one level */
                                iter->tnode = n;
                                iter->index = 0;
                                ++iter->depth;
                        }

                        return n;
                }

                /* Current node exhausted, pop back up */
                pkey = pn->key;
                pn = node_parent_rcu(pn);
                cindex = get_index(pkey, pn) + 1;
                --iter->depth;
        }

        /* record root node so further searches know we are done */
        iter->tnode = pn;
        iter->index = 0;

        return NULL;
}

static struct key_vector *fib_trie_get_first(struct fib_trie_iter *iter,
                                             struct trie *t)
{
        struct key_vector *n, *pn;

        if (!t)
                return NULL;

        pn = t->kv;
        n = rcu_dereference(pn->tnode[0]);
        if (!n)
                return NULL;

        if (IS_TNODE(n)) {
                iter->tnode = n;
                iter->index = 0;
                iter->depth = 1;
        } else {
                iter->tnode = pn;
                iter->index = 0;
                iter->depth = 0;
        }

        return n;
}

static void trie_collect_stats(struct trie *t, struct trie_stat *s)
{
        struct key_vector *n;
        struct fib_trie_iter iter;

        memset(s, 0, sizeof(*s));

        rcu_read_lock();
        for (n = fib_trie_get_first(&iter, t); n; n = fib_trie_get_next(&iter)) {
                if (IS_LEAF(n)) {
                        struct fib_alias *fa;

                        s->leaves++;
                        s->totdepth += iter.depth;
                        if (iter.depth > s->maxdepth)
                                s->maxdepth = iter.depth;

                        hlist_for_each_entry_rcu(fa, &n->leaf, fa_list)
                                ++s->prefixes;
                } else {
                        s->tnodes++;
                        if (n->bits < MAX_STAT_DEPTH)
                                s->nodesizes[n->bits]++;
                        s->nullpointers += tn_info(n)->empty_children;
                }
        }
        rcu_read_unlock();
}

/*
 *        This outputs /proc/net/fib_triestats
 */
static void trie_show_stats(struct seq_file *seq, struct trie_stat *stat)
{
        unsigned int i, max, pointers, bytes, avdepth;

        if (stat->leaves)
                avdepth = stat->totdepth*100 / stat->leaves;
        else
                avdepth = 0;

        seq_printf(seq, "\tAver depth:     %u.%02d\n",
                   avdepth / 100, avdepth % 100);
        seq_printf(seq, "\tMax depth:      %u\n", stat->maxdepth);

        seq_printf(seq, "\tLeaves:         %u\n", stat->leaves);
        bytes = LEAF_SIZE * stat->leaves;

        seq_printf(seq, "\tPrefixes:       %u\n", stat->prefixes);
        bytes += sizeof(struct fib_alias) * stat->prefixes;

        seq_printf(seq, "\tInternal nodes: %u\n\t", stat->tnodes);
        bytes += TNODE_SIZE(0) * stat->tnodes;

        max = MAX_STAT_DEPTH;
        while (max > 0 && stat->nodesizes[max-1] == 0)
                max--;

        pointers = 0;
        for (i = 1; i < max; i++)
                if (stat->nodesizes[i] != 0) {
                        seq_printf(seq, "  %u: %u",  i, stat->nodesizes[i]);
                        pointers += (1<<i) * stat->nodesizes[i];
                }
        seq_putc(seq, '\n');
        seq_printf(seq, "\tPointers: %u\n", pointers);

        bytes += sizeof(struct key_vector *) * pointers;
        seq_printf(seq, "Null ptrs: %u\n", stat->nullpointers);
        seq_printf(seq, "Total size: %u  kB\n", (bytes + 1023) / 1024);
}

#ifdef CONFIG_IP_FIB_TRIE_STATS
static void trie_show_usage(struct seq_file *seq,
                            const struct trie_use_stats __percpu *stats)
{
        struct trie_use_stats s = { 0 };
        int cpu;

        /* loop through all of the CPUs and gather up the stats */
        for_each_possible_cpu(cpu) {
                const struct trie_use_stats *pcpu = per_cpu_ptr(stats, cpu);

                s.gets += pcpu->gets;
                s.backtrack += pcpu->backtrack;
                s.semantic_match_passed += pcpu->semantic_match_passed;
                s.semantic_match_miss += pcpu->semantic_match_miss;
                s.null_node_hit += pcpu->null_node_hit;
                s.resize_node_skipped += pcpu->resize_node_skipped;
        }

        seq_printf(seq, "\nCounters:\n---------\n");
        seq_printf(seq, "gets = %u\n", s.gets);
        seq_printf(seq, "backtracks = %u\n", s.backtrack);
        seq_printf(seq, "semantic match passed = %u\n",
                   s.semantic_match_passed);
        seq_printf(seq, "semantic match miss = %u\n", s.semantic_match_miss);
        seq_printf(seq, "null node hit= %u\n", s.null_node_hit);
        seq_printf(seq, "skipped node resize = %u\n\n", s.resize_node_skipped);
}
#endif /*  CONFIG_IP_FIB_TRIE_STATS */

static void fib_table_print(struct seq_file *seq, struct fib_table *tb)
{
        if (tb->tb_id == RT_TABLE_LOCAL)
                seq_puts(seq, "Local:\n");
        else if (tb->tb_id == RT_TABLE_MAIN)
                seq_puts(seq, "Main:\n");
        else
                seq_printf(seq, "Id %d:\n", tb->tb_id);
}


static int fib_triestat_seq_show(struct seq_file *seq, void *v)
{
        struct net *net = seq->private;
        unsigned int h;

        seq_printf(seq,
                   "Basic info: size of leaf:"
                   " %zd bytes, size of tnode: %zd bytes.\n",
                   LEAF_SIZE, TNODE_SIZE(0));

        rcu_read_lock();
        for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
                struct hlist_head *head = &net->ipv4.fib_table_hash[h];
                struct fib_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb_hlist) {
                        struct trie *t = (struct trie *) tb->tb_data;
                        struct trie_stat stat;

                        if (!t)
                                continue;

                        fib_table_print(seq, tb);

                        trie_collect_stats(t, &stat);
                        trie_show_stats(seq, &stat);
#ifdef CONFIG_IP_FIB_TRIE_STATS
                        trie_show_usage(seq, t->stats);
#endif
                }
                cond_resched_rcu();
        }
        rcu_read_unlock();

        return 0;
}

static struct key_vector *fib_trie_get_idx(struct seq_file *seq, loff_t pos)
{
        struct fib_trie_iter *iter = seq->private;
        struct net *net = seq_file_net(seq);
        loff_t idx = 0;
        unsigned int h;

        for (h = 0; h < FIB_TABLE_HASHSZ; h++) {
                struct hlist_head *head = &net->ipv4.fib_table_hash[h];
                struct fib_table *tb;

                hlist_for_each_entry_rcu(tb, head, tb_hlist) {
                        struct key_vector *n;

                        for (n = fib_trie_get_first(iter,
                                                    (struct trie *) tb->tb_data);
                             n; n = fib_trie_get_next(iter))
                                if (pos == idx++) {
                                        iter->tb = tb;
                                        return n;
                                }
                }
        }

        return NULL;
}

static void *fib_trie_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(RCU)
{
        rcu_read_lock();
        return fib_trie_get_idx(seq, *pos);
}

static void *fib_trie_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct fib_trie_iter *iter = seq->private;
        struct net *net = seq_file_net(seq);
        struct fib_table *tb = iter->tb;
        struct hlist_node *tb_node;
        unsigned int h;
        struct key_vector *n;

        ++*pos;
        /* next node in same table */
        n = fib_trie_get_next(iter);
        if (n)
                return n;

        /* walk rest of this hash chain */
        h = tb->tb_id & (FIB_TABLE_HASHSZ - 1);
        while ((tb_node = rcu_dereference(hlist_next_rcu(&tb->tb_hlist)))) {
                tb = hlist_entry(tb_node, struct fib_table, tb_hlist);
                n = fib_trie_get_first(iter, (struct trie *) tb->tb_data);
                if (n)
                        goto found;
        }

        /* new hash chain */
        while (++h < FIB_TABLE_HASHSZ) {
                struct hlist_head *head = &net->ipv4.fib_table_hash[h];
                hlist_for_each_entry_rcu(tb, head, tb_hlist) {
                        n = fib_trie_get_first(iter, (struct trie *) tb->tb_data);
                        if (n)
                                goto found;
                }
        }
        return NULL;

found:
        iter->tb = tb;
        return n;
}

static void fib_trie_seq_stop(struct seq_file *seq, void *v)
        __releases(RCU)
{
        rcu_read_unlock();
}

static void seq_indent(struct seq_file *seq, int n)
{
        while (n-- > 0)
                seq_puts(seq, "   ");
}

static inline const char *rtn_scope(char *buf, size_t len, enum rt_scope_t s)
{
        switch (s) {
        case RT_SCOPE_UNIVERSE: return "universe";
        case RT_SCOPE_SITE:        return "site";
        case RT_SCOPE_LINK:        return "link";
        case RT_SCOPE_HOST:        return "host";
        case RT_SCOPE_NOWHERE:        return "nowhere";
        default:
                snprintf(buf, len, "scope=%d", s);
                return buf;
        }
}

static const char *const rtn_type_names[__RTN_MAX] = {
        [RTN_UNSPEC] = "UNSPEC",
        [RTN_UNICAST] = "UNICAST",
        [RTN_LOCAL] = "LOCAL",
        [RTN_BROADCAST] = "BROADCAST",
        [RTN_ANYCAST] = "ANYCAST",
        [RTN_MULTICAST] = "MULTICAST",
        [RTN_BLACKHOLE] = "BLACKHOLE",
        [RTN_UNREACHABLE] = "UNREACHABLE",
        [RTN_PROHIBIT] = "PROHIBIT",
        [RTN_THROW] = "THROW",
        [RTN_NAT] = "NAT",
        [RTN_XRESOLVE] = "XRESOLVE",
};

static inline const char *rtn_type(char *buf, size_t len, unsigned int t)
{
        if (t < __RTN_MAX && rtn_type_names[t])
                return rtn_type_names[t];
        snprintf(buf, len, "type %u", t);
        return buf;
}

/* Pretty print the trie */
static int fib_trie_seq_show(struct seq_file *seq, void *v)
{
        const struct fib_trie_iter *iter = seq->private;
        struct key_vector *n = v;

        if (IS_TRIE(node_parent_rcu(n)))
                fib_table_print(seq, iter->tb);

        if (IS_TNODE(n)) {
                __be32 prf = htonl(n->key);

                seq_indent(seq, iter->depth-1);
                seq_printf(seq, "  +-- %pI4/%zu %u %u %u\n",
                           &prf, KEYLENGTH - n->pos - n->bits, n->bits,
                           tn_info(n)->full_children,
                           tn_info(n)->empty_children);
        } else {
                __be32 val = htonl(n->key);
                struct fib_alias *fa;

                seq_indent(seq, iter->depth);
                seq_printf(seq, "  |-- %pI4\n", &val);

                hlist_for_each_entry_rcu(fa, &n->leaf, fa_list) {
                        char buf1[32], buf2[32];

                        seq_indent(seq, iter->depth + 1);
                        seq_printf(seq, "  /%zu %s %s",
                                   KEYLENGTH - fa->fa_slen,
                                   rtn_scope(buf1, sizeof(buf1),
                                             fa->fa_info->fib_scope),
                                   rtn_type(buf2, sizeof(buf2),
                                            fa->fa_type));
                        if (fa->fa_dscp)
                                seq_printf(seq, " tos=%d",
                                           inet_dscp_to_dsfield(fa->fa_dscp));
                        seq_putc(seq, '\n');
                }
        }

        return 0;
}

static const struct seq_operations fib_trie_seq_ops = {
        .start  = fib_trie_seq_start,
        .next   = fib_trie_seq_next,
        .stop   = fib_trie_seq_stop,
        .show   = fib_trie_seq_show,
};

struct fib_route_iter {
        struct seq_net_private p;
        struct fib_table *main_tb;
        struct key_vector *tnode;
        loff_t        pos;
        t_key        key;
};

static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
                                            loff_t pos)
{
        struct key_vector *l, **tp = &iter->tnode;
        t_key key;

        /* use cached location of previously found key */
        if (iter->pos > 0 && pos >= iter->pos) {
                key = iter->key;
        } else {
                iter->pos = 1;
                key = 0;
        }

        pos -= iter->pos;

        while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
                key = l->key + 1;
                iter->pos++;
                l = NULL;

                /* handle unlikely case of a key wrap */
                if (!key)
                        break;
        }

        if (l)
                iter->key = l->key;        /* remember it */
        else
                iter->pos = 0;                /* forget it */

        return l;
}

static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(RCU)
{
        struct fib_route_iter *iter = seq->private;
        struct fib_table *tb;
        struct trie *t;

        rcu_read_lock();

        tb = fib_get_table(seq_file_net(seq), RT_TABLE_MAIN);
        if (!tb)
                return NULL;

        iter->main_tb = tb;
        t = (struct trie *)tb->tb_data;
        iter->tnode = t->kv;

        if (*pos != 0)
                return fib_route_get_idx(iter, *pos);

        iter->pos = 0;
        iter->key = KEY_MAX;

        return SEQ_START_TOKEN;
}

static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct fib_route_iter *iter = seq->private;
        struct key_vector *l = NULL;
        t_key key = iter->key + 1;

        ++*pos;

        /* only allow key of 0 for start of sequence */
        if ((v == SEQ_START_TOKEN) || key)
                l = leaf_walk_rcu(&iter->tnode, key);

        if (l) {
                iter->key = l->key;
                iter->pos++;
        } else {
                iter->pos = 0;
        }

        return l;
}

static void fib_route_seq_stop(struct seq_file *seq, void *v)
        __releases(RCU)
{
        rcu_read_unlock();
}

static unsigned int fib_flag_trans(int type, __be32 mask, struct fib_info *fi)
{
        unsigned int flags = 0;

        if (type == RTN_UNREACHABLE || type == RTN_PROHIBIT)
                flags = RTF_REJECT;
        if (fi) {
                const struct fib_nh_common *nhc = fib_info_nhc(fi, 0);

                if (nhc->nhc_gw.ipv4)
                        flags |= RTF_GATEWAY;
        }
        if (mask == htonl(0xFFFFFFFF))
                flags |= RTF_HOST;
        flags |= RTF_UP;
        return flags;
}

/*
 *        This outputs /proc/net/route.
 *        The format of the file is not supposed to be changed
 *        and needs to be same as fib_hash output to avoid breaking
 *        legacy utilities
 */
static int fib_route_seq_show(struct seq_file *seq, void *v)
{
        struct fib_route_iter *iter = seq->private;
        struct fib_table *tb = iter->main_tb;
        struct fib_alias *fa;
        struct key_vector *l = v;
        __be32 prefix;

        if (v == SEQ_START_TOKEN) {
                seq_printf(seq, "%-127s\n", "Iface\tDestination\tGateway "
                           "\tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU"
                           "\tWindow\tIRTT");
                return 0;
        }

        prefix = htonl(l->key);

        hlist_for_each_entry_rcu(fa, &l->leaf, fa_list) {
                struct fib_info *fi = fa->fa_info;
                __be32 mask = inet_make_mask(KEYLENGTH - fa->fa_slen);
                unsigned int flags = fib_flag_trans(fa->fa_type, mask, fi);

                if ((fa->fa_type == RTN_BROADCAST) ||
                    (fa->fa_type == RTN_MULTICAST))
                        continue;

                if (fa->tb_id != tb->tb_id)
                        continue;

                seq_setwidth(seq, 127);

                if (fi) {
                        struct fib_nh_common *nhc = fib_info_nhc(fi, 0);
                        __be32 gw = 0;

                        if (nhc->nhc_gw_family == AF_INET)
                                gw = nhc->nhc_gw.ipv4;

                        seq_printf(seq,
                                   "%s\t%08X\t%08X\t%04X\t%d\t%u\t"
                                   "%u\t%08X\t%d\t%u\t%u",
                                   nhc->nhc_dev ? nhc->nhc_dev->name : "*",
                                   prefix, gw, flags, 0, 0,
                                   fi->fib_priority,
                                   mask,
                                   (fi->fib_advmss ?
                                    fi->fib_advmss + 40 : 0),
                                   fi->fib_window,
                                   fi->fib_rtt >> 3);
                } else {
                        seq_printf(seq,
                                   "*\t%08X\t%08X\t%04X\t%d\t%u\t"
                                   "%u\t%08X\t%d\t%u\t%u",
                                   prefix, 0, flags, 0, 0, 0,
                                   mask, 0, 0, 0);
                }
                seq_pad(seq, '\n');
        }

        return 0;
}

static const struct seq_operations fib_route_seq_ops = {
        .start  = fib_route_seq_start,
        .next   = fib_route_seq_next,
        .stop   = fib_route_seq_stop,
        .show   = fib_route_seq_show,
};

int __net_init fib_proc_init(struct net *net)
{
        if (!proc_create_net("fib_trie", 0444, net->proc_net, &fib_trie_seq_ops,
                        sizeof(struct fib_trie_iter)))
                goto out1;

        if (!proc_create_net_single("fib_triestat", 0444, net->proc_net,
                        fib_triestat_seq_show, NULL))
                goto out2;

        if (!proc_create_net("route", 0444, net->proc_net, &fib_route_seq_ops,
                        sizeof(struct fib_route_iter)))
                goto out3;

        return 0;

out3:
        remove_proc_entry("fib_triestat", net->proc_net);
out2:
        remove_proc_entry("fib_trie", net->proc_net);
out1:
        return -ENOMEM;
}

void __net_exit fib_proc_exit(struct net *net)
{
        remove_proc_entry("fib_trie", net->proc_net);
        remove_proc_entry("fib_triestat", net->proc_net);
        remove_proc_entry("route", net->proc_net);
}

#endif /* CONFIG_PROC_FS */






























  297 














  259 







    2 



    2 


  264 
















  263 





  263 











   23 










    1 






   23 





    5 




    3 

   12 





















    1 







   23 

























   32 










   33 



   17 

    8 

    1 

    2 

    1 

    1 


    1 






    1 



    1 










    4 





    3 



    1 

    2 
    3 























    3 



    6 


    1 


    1 











    5 








    2 







    2 









    1 





    1 




    1 








    1 









    6 





    6 







  290 


































  285 










    9 










    1 



    1 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
// SPDX-License-Identifier: GPL-2.0
#include <linux/mount.h>
#include <linux/pseudo_fs.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/proc_fs.h>
#include <linux/proc_ns.h>
#include <linux/magic.h>
#include <linux/ktime.h>
#include <linux/seq_file.h>
#include <linux/pid_namespace.h>
#include <linux/user_namespace.h>
#include <linux/nsfs.h>
#include <linux/uaccess.h>
#include <linux/mnt_namespace.h>

#include "mount.h"
#include "internal.h"

static struct vfsmount *nsfs_mnt;

static long ns_ioctl(struct file *filp, unsigned int ioctl,
                        unsigned long arg);
static const struct file_operations ns_file_operations = {
        .unlocked_ioctl = ns_ioctl,
        .compat_ioctl   = compat_ptr_ioctl,
};

static char *ns_dname(struct dentry *dentry, char *buffer, int buflen)
{
        struct inode *inode = d_inode(dentry);
        struct ns_common *ns = inode->i_private;
        const struct proc_ns_operations *ns_ops = ns->ops;

        return dynamic_dname(buffer, buflen, "%s:[%lu]",
                ns_ops->name, inode->i_ino);
}

const struct dentry_operations ns_dentry_operations = {
        .d_dname        = ns_dname,
        .d_prune        = stashed_dentry_prune,
};

static void nsfs_evict(struct inode *inode)
{
        struct ns_common *ns = inode->i_private;
        clear_inode(inode);
        ns->ops->put(ns);
}

int ns_get_path_cb(struct path *path, ns_get_path_helper_t *ns_get_cb,
                     void *private_data)
{
        struct ns_common *ns;

        ns = ns_get_cb(private_data);
        if (!ns)
                return -ENOENT;

        return path_from_stashed(&ns->stashed, nsfs_mnt, ns, path);
}

struct ns_get_path_task_args {
        const struct proc_ns_operations *ns_ops;
        struct task_struct *task;
};

static struct ns_common *ns_get_path_task(void *private_data)
{
        struct ns_get_path_task_args *args = private_data;

        return args->ns_ops->get(args->task);
}

int ns_get_path(struct path *path, struct task_struct *task,
                  const struct proc_ns_operations *ns_ops)
{
        struct ns_get_path_task_args args = {
                .ns_ops        = ns_ops,
                .task        = task,
        };

        return ns_get_path_cb(path, ns_get_path_task, &args);
}

/**
 * open_namespace - open a namespace
 * @ns: the namespace to open
 *
 * This will consume a reference to @ns indendent of success or failure.
 *
 * Return: A file descriptor on success or a negative error code on failure.
 */
int open_namespace(struct ns_common *ns)
{
        struct path path __free(path_put) = {};
        struct file *f;
        int err;

        /* call first to consume reference */
        err = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path);
        if (err < 0)
                return err;

        CLASS(get_unused_fd, fd)(O_CLOEXEC);
        if (fd < 0)
                return fd;

        f = dentry_open(&path, O_RDONLY, current_cred());
        if (IS_ERR(f))
                return PTR_ERR(f);

        fd_install(fd, f);
        return take_fd(fd);
}

int open_related_ns(struct ns_common *ns,
                   struct ns_common *(*get_ns)(struct ns_common *ns))
{
        struct ns_common *relative;

        relative = get_ns(ns);
        if (IS_ERR(relative))
                return PTR_ERR(relative);

        return open_namespace(relative);
}
EXPORT_SYMBOL_GPL(open_related_ns);

static int copy_ns_info_to_user(const struct mnt_namespace *mnt_ns,
                                struct mnt_ns_info __user *uinfo, size_t usize,
                                struct mnt_ns_info *kinfo)
{
        /*
         * If userspace and the kernel have the same struct size it can just
         * be copied. If userspace provides an older struct, only the bits that
         * userspace knows about will be copied. If userspace provides a new
         * struct, only the bits that the kernel knows aobut will be copied and
         * the size value will be set to the size the kernel knows about.
         */
        kinfo->size                = min(usize, sizeof(*kinfo));
        kinfo->mnt_ns_id        = mnt_ns->seq;
        kinfo->nr_mounts        = READ_ONCE(mnt_ns->nr_mounts);
        /* Subtract the root mount of the mount namespace. */
        if (kinfo->nr_mounts)
                kinfo->nr_mounts--;

        if (copy_to_user(uinfo, kinfo, kinfo->size))
                return -EFAULT;

        return 0;
}

static bool nsfs_ioctl_valid(unsigned int cmd)
{
        switch (cmd) {
        case NS_GET_USERNS:
        case NS_GET_PARENT:
        case NS_GET_NSTYPE:
        case NS_GET_OWNER_UID:
        case NS_GET_MNTNS_ID:
        case NS_GET_PID_FROM_PIDNS:
        case NS_GET_TGID_FROM_PIDNS:
        case NS_GET_PID_IN_PIDNS:
        case NS_GET_TGID_IN_PIDNS:
                return (_IOC_TYPE(cmd) == _IOC_TYPE(cmd));
        }

        /* Extensible ioctls require some extra handling. */
        switch (_IOC_NR(cmd)) {
        case _IOC_NR(NS_MNT_GET_INFO):
        case _IOC_NR(NS_MNT_GET_NEXT):
        case _IOC_NR(NS_MNT_GET_PREV):
                return (_IOC_TYPE(cmd) == _IOC_TYPE(cmd));
        }

        return false;
}

static long ns_ioctl(struct file *filp, unsigned int ioctl,
                        unsigned long arg)
{
        struct user_namespace *user_ns;
        struct pid_namespace *pid_ns;
        struct task_struct *tsk;
        struct ns_common *ns;
        struct mnt_namespace *mnt_ns;
        bool previous = false;
        uid_t __user *argp;
        uid_t uid;
        int ret;

        if (!nsfs_ioctl_valid(ioctl))
                return -ENOIOCTLCMD;

        ns = get_proc_ns(file_inode(filp));
        switch (ioctl) {
        case NS_GET_USERNS:
                return open_related_ns(ns, ns_get_owner);
        case NS_GET_PARENT:
                if (!ns->ops->get_parent)
                        return -EINVAL;
                return open_related_ns(ns, ns->ops->get_parent);
        case NS_GET_NSTYPE:
                return ns->ops->type;
        case NS_GET_OWNER_UID:
                if (ns->ops->type != CLONE_NEWUSER)
                        return -EINVAL;
                user_ns = container_of(ns, struct user_namespace, ns);
                argp = (uid_t __user *) arg;
                uid = from_kuid_munged(current_user_ns(), user_ns->owner);
                return put_user(uid, argp);
        case NS_GET_MNTNS_ID: {
                __u64 __user *idp;
                __u64 id;

                if (ns->ops->type != CLONE_NEWNS)
                        return -EINVAL;

                mnt_ns = container_of(ns, struct mnt_namespace, ns);
                idp = (__u64 __user *)arg;
                id = mnt_ns->seq;
                return put_user(id, idp);
        }
        case NS_GET_PID_FROM_PIDNS:
                fallthrough;
        case NS_GET_TGID_FROM_PIDNS:
                fallthrough;
        case NS_GET_PID_IN_PIDNS:
                fallthrough;
        case NS_GET_TGID_IN_PIDNS: {
                if (ns->ops->type != CLONE_NEWPID)
                        return -EINVAL;

                ret = -ESRCH;
                pid_ns = container_of(ns, struct pid_namespace, ns);

                guard(rcu)();

                if (ioctl == NS_GET_PID_IN_PIDNS ||
                    ioctl == NS_GET_TGID_IN_PIDNS)
                        tsk = find_task_by_vpid(arg);
                else
                        tsk = find_task_by_pid_ns(arg, pid_ns);
                if (!tsk)
                        break;

                switch (ioctl) {
                case NS_GET_PID_FROM_PIDNS:
                        ret = task_pid_vnr(tsk);
                        break;
                case NS_GET_TGID_FROM_PIDNS:
                        ret = task_tgid_vnr(tsk);
                        break;
                case NS_GET_PID_IN_PIDNS:
                        ret = task_pid_nr_ns(tsk, pid_ns);
                        break;
                case NS_GET_TGID_IN_PIDNS:
                        ret = task_tgid_nr_ns(tsk, pid_ns);
                        break;
                default:
                        ret = 0;
                        break;
                }

                if (!ret)
                        ret = -ESRCH;
                return ret;
        }
        }

        /* extensible ioctls */
        switch (_IOC_NR(ioctl)) {
        case _IOC_NR(NS_MNT_GET_INFO): {
                struct mnt_ns_info kinfo = {};
                struct mnt_ns_info __user *uinfo = (struct mnt_ns_info __user *)arg;
                size_t usize = _IOC_SIZE(ioctl);

                if (ns->ops->type != CLONE_NEWNS)
                        return -EINVAL;

                if (!uinfo)
                        return -EINVAL;

                if (usize < MNT_NS_INFO_SIZE_VER0)
                        return -EINVAL;

                return copy_ns_info_to_user(to_mnt_ns(ns), uinfo, usize, &kinfo);
        }
        case _IOC_NR(NS_MNT_GET_PREV):
                previous = true;
                fallthrough;
        case _IOC_NR(NS_MNT_GET_NEXT): {
                struct mnt_ns_info kinfo = {};
                struct mnt_ns_info __user *uinfo = (struct mnt_ns_info __user *)arg;
                struct path path __free(path_put) = {};
                struct file *f __free(fput) = NULL;
                size_t usize = _IOC_SIZE(ioctl);

                if (ns->ops->type != CLONE_NEWNS)
                        return -EINVAL;

                if (usize < MNT_NS_INFO_SIZE_VER0)
                        return -EINVAL;

                mnt_ns = get_sequential_mnt_ns(to_mnt_ns(ns), previous);
                if (IS_ERR(mnt_ns))
                        return PTR_ERR(mnt_ns);

                ns = to_ns_common(mnt_ns);
                /* Transfer ownership of @mnt_ns reference to @path. */
                ret = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path);
                if (ret)
                        return ret;

                CLASS(get_unused_fd, fd)(O_CLOEXEC);
                if (fd < 0)
                        return fd;

                f = dentry_open(&path, O_RDONLY, current_cred());
                if (IS_ERR(f))
                        return PTR_ERR(f);

                if (uinfo) {
                        /*
                         * If @uinfo is passed return all information about the
                         * mount namespace as well.
                         */
                        ret = copy_ns_info_to_user(to_mnt_ns(ns), uinfo, usize, &kinfo);
                        if (ret)
                                return ret;
                }

                /* Transfer reference of @f to caller's fdtable. */
                fd_install(fd, no_free_ptr(f));
                /* File descriptor is live so hand it off to the caller. */
                return take_fd(fd);
        }
        default:
                ret = -ENOTTY;
        }

        return ret;
}

int ns_get_name(char *buf, size_t size, struct task_struct *task,
                        const struct proc_ns_operations *ns_ops)
{
        struct ns_common *ns;
        int res = -ENOENT;
        const char *name;
        ns = ns_ops->get(task);
        if (ns) {
                name = ns_ops->real_ns_name ? : ns_ops->name;
                res = snprintf(buf, size, "%s:[%u]", name, ns->inum);
                ns_ops->put(ns);
        }
        return res;
}

bool proc_ns_file(const struct file *file)
{
        return file->f_op == &ns_file_operations;
}

/**
 * ns_match() - Returns true if current namespace matches dev/ino provided.
 * @ns: current namespace
 * @dev: dev_t from nsfs that will be matched against current nsfs
 * @ino: ino_t from nsfs that will be matched against current nsfs
 *
 * Return: true if dev and ino matches the current nsfs.
 */
bool ns_match(const struct ns_common *ns, dev_t dev, ino_t ino)
{
        return (ns->inum == ino) && (nsfs_mnt->mnt_sb->s_dev == dev);
}


static int nsfs_show_path(struct seq_file *seq, struct dentry *dentry)
{
        struct inode *inode = d_inode(dentry);
        const struct ns_common *ns = inode->i_private;
        const struct proc_ns_operations *ns_ops = ns->ops;

        seq_printf(seq, "%s:[%lu]", ns_ops->name, inode->i_ino);
        return 0;
}

static const struct super_operations nsfs_ops = {
        .statfs = simple_statfs,
        .evict_inode = nsfs_evict,
        .show_path = nsfs_show_path,
};

static int nsfs_init_inode(struct inode *inode, void *data)
{
        struct ns_common *ns = data;

        inode->i_private = data;
        inode->i_mode |= S_IRUGO;
        inode->i_fop = &ns_file_operations;
        inode->i_ino = ns->inum;
        return 0;
}

static void nsfs_put_data(void *data)
{
        struct ns_common *ns = data;
        ns->ops->put(ns);
}

static const struct stashed_operations nsfs_stashed_ops = {
        .init_inode = nsfs_init_inode,
        .put_data = nsfs_put_data,
};

static int nsfs_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx = init_pseudo(fc, NSFS_MAGIC);
        if (!ctx)
                return -ENOMEM;
        ctx->ops = &nsfs_ops;
        ctx->dops = &ns_dentry_operations;
        fc->s_fs_info = (void *)&nsfs_stashed_ops;
        return 0;
}

static struct file_system_type nsfs = {
        .name = "nsfs",
        .init_fs_context = nsfs_init_fs_context,
        .kill_sb = kill_anon_super,
};

void __init nsfs_init(void)
{
        nsfs_mnt = kern_mount(&nsfs);
        if (IS_ERR(nsfs_mnt))
                panic("can't set nsfs up\n");
        nsfs_mnt->mnt_sb->s_flags &= ~SB_NOUSER;
}




























    2 












    2 

























































































    3 









    3 










































































    9 
    7 
    2 


    6 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2012-2016 Pablo Neira Ayuso <pablo@netfilter.org>
 */

#include <linux/init.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>

#define nft_objref_priv(expr)        *((struct nft_object **)nft_expr_priv(expr))

void nft_objref_eval(const struct nft_expr *expr,
                     struct nft_regs *regs,
                     const struct nft_pktinfo *pkt)
{
        struct nft_object *obj = nft_objref_priv(expr);

        obj->ops->eval(obj, regs, pkt);
}

static int nft_objref_init(const struct nft_ctx *ctx,
                           const struct nft_expr *expr,
                           const struct nlattr * const tb[])
{
        struct nft_object *obj = nft_objref_priv(expr);
        u8 genmask = nft_genmask_next(ctx->net);
        u32 objtype;

        if (!tb[NFTA_OBJREF_IMM_NAME] ||
            !tb[NFTA_OBJREF_IMM_TYPE])
                return -EINVAL;

        objtype = ntohl(nla_get_be32(tb[NFTA_OBJREF_IMM_TYPE]));
        obj = nft_obj_lookup(ctx->net, ctx->table,
                             tb[NFTA_OBJREF_IMM_NAME], objtype,
                             genmask);
        if (IS_ERR(obj))
                return -ENOENT;

        if (!nft_use_inc(&obj->use))
                return -EMFILE;

        nft_objref_priv(expr) = obj;

        return 0;
}

static int nft_objref_dump(struct sk_buff *skb,
                           const struct nft_expr *expr, bool reset)
{
        const struct nft_object *obj = nft_objref_priv(expr);

        if (nla_put_string(skb, NFTA_OBJREF_IMM_NAME, obj->key.name) ||
            nla_put_be32(skb, NFTA_OBJREF_IMM_TYPE,
                         htonl(obj->ops->type->type)))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -1;
}

static void nft_objref_deactivate(const struct nft_ctx *ctx,
                                  const struct nft_expr *expr,
                                  enum nft_trans_phase phase)
{
        struct nft_object *obj = nft_objref_priv(expr);

        if (phase == NFT_TRANS_COMMIT)
                return;

        nft_use_dec(&obj->use);
}

static void nft_objref_activate(const struct nft_ctx *ctx,
                                const struct nft_expr *expr)
{
        struct nft_object *obj = nft_objref_priv(expr);

        nft_use_inc_restore(&obj->use);
}

static const struct nft_expr_ops nft_objref_ops = {
        .type                = &nft_objref_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_object *)),
        .eval                = nft_objref_eval,
        .init                = nft_objref_init,
        .activate        = nft_objref_activate,
        .deactivate        = nft_objref_deactivate,
        .dump                = nft_objref_dump,
        .reduce                = NFT_REDUCE_READONLY,
};

struct nft_objref_map {
        struct nft_set                *set;
        u8                        sreg;
        struct nft_set_binding        binding;
};

void nft_objref_map_eval(const struct nft_expr *expr,
                         struct nft_regs *regs,
                         const struct nft_pktinfo *pkt)
{
        struct nft_objref_map *priv = nft_expr_priv(expr);
        const struct nft_set *set = priv->set;
        struct net *net = nft_net(pkt);
        const struct nft_set_ext *ext;
        struct nft_object *obj;
        bool found;

        found = nft_set_do_lookup(net, set, &regs->data[priv->sreg], &ext);
        if (!found) {
                ext = nft_set_catchall_lookup(net, set);
                if (!ext) {
                        regs->verdict.code = NFT_BREAK;
                        return;
                }
        }
        obj = *nft_set_ext_obj(ext);
        obj->ops->eval(obj, regs, pkt);
}

static int nft_objref_map_init(const struct nft_ctx *ctx,
                               const struct nft_expr *expr,
                               const struct nlattr * const tb[])
{
        struct nft_objref_map *priv = nft_expr_priv(expr);
        u8 genmask = nft_genmask_next(ctx->net);
        struct nft_set *set;
        int err;

        set = nft_set_lookup_global(ctx->net, ctx->table,
                                    tb[NFTA_OBJREF_SET_NAME],
                                    tb[NFTA_OBJREF_SET_ID], genmask);
        if (IS_ERR(set))
                return PTR_ERR(set);

        if (!(set->flags & NFT_SET_OBJECT))
                return -EINVAL;

        err = nft_parse_register_load(ctx, tb[NFTA_OBJREF_SET_SREG], &priv->sreg,
                                      set->klen);
        if (err < 0)
                return err;

        priv->binding.flags = set->flags & NFT_SET_OBJECT;

        err = nf_tables_bind_set(ctx, set, &priv->binding);
        if (err < 0)
                return err;

        priv->set = set;
        return 0;
}

static int nft_objref_map_dump(struct sk_buff *skb,
                               const struct nft_expr *expr, bool reset)
{
        const struct nft_objref_map *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_OBJREF_SET_SREG, priv->sreg) ||
            nla_put_string(skb, NFTA_OBJREF_SET_NAME, priv->set->name))
                goto nla_put_failure;

        return 0;

nla_put_failure:
        return -1;
}

static void nft_objref_map_deactivate(const struct nft_ctx *ctx,
                                      const struct nft_expr *expr,
                                      enum nft_trans_phase phase)
{
        struct nft_objref_map *priv = nft_expr_priv(expr);

        nf_tables_deactivate_set(ctx, priv->set, &priv->binding, phase);
}

static void nft_objref_map_activate(const struct nft_ctx *ctx,
                                    const struct nft_expr *expr)
{
        struct nft_objref_map *priv = nft_expr_priv(expr);

        nf_tables_activate_set(ctx, priv->set);
}

static void nft_objref_map_destroy(const struct nft_ctx *ctx,
                                   const struct nft_expr *expr)
{
        struct nft_objref_map *priv = nft_expr_priv(expr);

        nf_tables_destroy_set(ctx, priv->set);
}

static const struct nft_expr_ops nft_objref_map_ops = {
        .type                = &nft_objref_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_objref_map)),
        .eval                = nft_objref_map_eval,
        .init                = nft_objref_map_init,
        .activate        = nft_objref_map_activate,
        .deactivate        = nft_objref_map_deactivate,
        .destroy        = nft_objref_map_destroy,
        .dump                = nft_objref_map_dump,
        .reduce                = NFT_REDUCE_READONLY,
};

static const struct nft_expr_ops *
nft_objref_select_ops(const struct nft_ctx *ctx,
                      const struct nlattr * const tb[])
{
        if (tb[NFTA_OBJREF_SET_SREG] &&
            (tb[NFTA_OBJREF_SET_NAME] ||
             tb[NFTA_OBJREF_SET_ID]))
                return &nft_objref_map_ops;
        else if (tb[NFTA_OBJREF_IMM_NAME] &&
                 tb[NFTA_OBJREF_IMM_TYPE])
                return &nft_objref_ops;

        return ERR_PTR(-EOPNOTSUPP);
}

static const struct nla_policy nft_objref_policy[NFTA_OBJREF_MAX + 1] = {
        [NFTA_OBJREF_IMM_NAME]        = { .type = NLA_STRING,
                                    .len = NFT_OBJ_MAXNAMELEN - 1 },
        [NFTA_OBJREF_IMM_TYPE]        = { .type = NLA_U32 },
        [NFTA_OBJREF_SET_SREG]        = { .type = NLA_U32 },
        [NFTA_OBJREF_SET_NAME]        = { .type = NLA_STRING,
                                    .len = NFT_SET_MAXNAMELEN - 1 },
        [NFTA_OBJREF_SET_ID]        = { .type = NLA_U32 },
};

struct nft_expr_type nft_objref_type __read_mostly = {
        .name                = "objref",
        .select_ops        = nft_objref_select_ops,
        .policy                = nft_objref_policy,
        .maxattr        = NFTA_OBJREF_MAX,
        .owner                = THIS_MODULE,
};
















































    8 

    4 





    5 





    4 


    3 
















   28 

    1 


   26 


   15 





   15 




   15 







   29 












   29 
   29 


   29 
   29 



   28 







   29 






   15 
   14 

   28 
    4 
   26 


    2 


    9 




   15 
   15 
   15 
    1 






   15 

    8 







    7 
    7 



   15 


































   20 












   20 








   18 
    8 
    8 

    8 
    8 




   12 


   12 






   12 
































   12 

   12 



   12 






    3 







    3 






    1 







    1 
















































































    1 
    1 







    1 
    1 







    1 
    1 






















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *        IPV6 GSO/GRO offload support
 *        Linux INET6 implementation
 */

#include <linux/kernel.h>
#include <linux/socket.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>
#include <linux/printk.h>

#include <net/protocol.h>
#include <net/ipv6.h>
#include <net/inet_common.h>
#include <net/tcp.h>
#include <net/udp.h>
#include <net/gro.h>
#include <net/gso.h>

#include "ip6_offload.h"

/* All GRO functions are always builtin, except UDP over ipv6, which lays in
 * ipv6 module, as it depends on UDPv6 lookup function, so we need special care
 * when ipv6 is built as a module
 */
#if IS_BUILTIN(CONFIG_IPV6)
#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__)
#else
#define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
#endif

#define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)        \
({                                                                \
        unlikely(gro_recursion_inc_test(skb)) ?                        \
                NAPI_GRO_CB(skb)->flush |= 1, NULL :                \
                INDIRECT_CALL_L4(cb, f2, f1, head, skb);        \
})

static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto)
{
        const struct net_offload *ops = NULL;
        struct ipv6_opt_hdr *opth;

        for (;;) {
                int len;

                ops = rcu_dereference(inet6_offloads[proto]);

                if (unlikely(!ops))
                        break;

                if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
                        break;

                opth = skb_gro_header(skb, off + sizeof(*opth), off);
                if (unlikely(!opth))
                        break;

                len = ipv6_optlen(opth);

                opth = skb_gro_header(skb, off + len, off);
                if (unlikely(!opth))
                        break;
                proto = opth->nexthdr;

                off += len;
        }

        skb_gro_pull(skb, off - skb_gro_receive_network_offset(skb));
        return proto;
}

static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
{
        const struct net_offload *ops = NULL;

        for (;;) {
                struct ipv6_opt_hdr *opth;
                int len;

                ops = rcu_dereference(inet6_offloads[proto]);

                if (unlikely(!ops))
                        break;

                if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
                        break;

                if (unlikely(!pskb_may_pull(skb, 8)))
                        break;

                opth = (void *)skb->data;
                len = ipv6_optlen(opth);

                if (unlikely(!pskb_may_pull(skb, len)))
                        break;

                opth = (void *)skb->data;
                proto = opth->nexthdr;
                __skb_pull(skb, len);
        }

        return proto;
}

static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
        netdev_features_t features)
{
        struct sk_buff *segs = ERR_PTR(-EINVAL);
        struct ipv6hdr *ipv6h;
        const struct net_offload *ops;
        int proto, err;
        struct frag_hdr *fptr;
        unsigned int payload_len;
        u8 *prevhdr;
        int offset = 0;
        bool encap, udpfrag;
        int nhoff;
        bool gso_partial;

        skb_reset_network_header(skb);
        err = ipv6_hopopt_jumbo_remove(skb);
        if (err)
                return ERR_PTR(err);
        nhoff = skb_network_header(skb) - skb_mac_header(skb);
        if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
                goto out;

        encap = SKB_GSO_CB(skb)->encap_level > 0;
        if (encap)
                features &= skb->dev->hw_enc_features;
        SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);

        ipv6h = ipv6_hdr(skb);
        __skb_pull(skb, sizeof(*ipv6h));
        segs = ERR_PTR(-EPROTONOSUPPORT);

        proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);

        if (skb->encapsulation &&
            skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))
                udpfrag = proto == IPPROTO_UDP && encap &&
                          (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
        else
                udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&
                          (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);

        ops = rcu_dereference(inet6_offloads[proto]);
        if (likely(ops && ops->callbacks.gso_segment)) {
                skb_reset_transport_header(skb);
                segs = ops->callbacks.gso_segment(skb, features);
                if (!segs)
                        skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
        }

        if (IS_ERR_OR_NULL(segs))
                goto out;

        gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);

        for (skb = segs; skb; skb = skb->next) {
                ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
                if (gso_partial && skb_is_gso(skb))
                        payload_len = skb_shinfo(skb)->gso_size +
                                      SKB_GSO_CB(skb)->data_offset +
                                      skb->head - (unsigned char *)(ipv6h + 1);
                else
                        payload_len = skb->len - nhoff - sizeof(*ipv6h);
                ipv6h->payload_len = htons(payload_len);
                skb->network_header = (u8 *)ipv6h - skb->head;
                skb_reset_mac_len(skb);

                if (udpfrag) {
                        int err = ip6_find_1stfragopt(skb, &prevhdr);
                        if (err < 0) {
                                kfree_skb_list(segs);
                                return ERR_PTR(err);
                        }
                        fptr = (struct frag_hdr *)((u8 *)ipv6h + err);
                        fptr->frag_off = htons(offset);
                        if (skb->next)
                                fptr->frag_off |= htons(IP6_MF);
                        offset += (ntohs(ipv6h->payload_len) -
                                   sizeof(struct frag_hdr));
                }
                if (encap)
                        skb_reset_inner_headers(skb);
        }

out:
        return segs;
}

/* Return the total length of all the extension hdrs, following the same
 * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
 */
static int ipv6_exthdrs_len(struct ipv6hdr *iph,
                            const struct net_offload **opps)
{
        struct ipv6_opt_hdr *opth = (void *)iph;
        int len = 0, proto, optlen = sizeof(*iph);

        proto = iph->nexthdr;
        for (;;) {
                *opps = rcu_dereference(inet6_offloads[proto]);
                if (unlikely(!(*opps)))
                        break;
                if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR))
                        break;

                opth = (void *)opth + optlen;
                optlen = ipv6_optlen(opth);
                len += optlen;
                proto = opth->nexthdr;
        }
        return len;
}

INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
                                                         struct sk_buff *skb)
{
        const struct net_offload *ops;
        struct sk_buff *pp = NULL;
        struct sk_buff *p;
        struct ipv6hdr *iph;
        unsigned int nlen;
        unsigned int hlen;
        unsigned int off;
        u16 flush = 1;
        int proto;

        off = skb_gro_offset(skb);
        hlen = off + sizeof(*iph);
        iph = skb_gro_header(skb, hlen, off);
        if (unlikely(!iph))
                goto out;

        NAPI_GRO_CB(skb)->network_offsets[NAPI_GRO_CB(skb)->encap_mark] = off;

        flush += ntohs(iph->payload_len) != skb->len - hlen;

        proto = iph->nexthdr;
        ops = rcu_dereference(inet6_offloads[proto]);
        if (!ops || !ops->callbacks.gro_receive) {
                proto = ipv6_gro_pull_exthdrs(skb, hlen, proto);

                ops = rcu_dereference(inet6_offloads[proto]);
                if (!ops || !ops->callbacks.gro_receive)
                        goto out;

                iph = skb_gro_network_header(skb);
        } else {
                skb_gro_pull(skb, sizeof(*iph));
        }

        skb_set_transport_header(skb, skb_gro_offset(skb));

        NAPI_GRO_CB(skb)->proto = proto;

        flush--;
        nlen = skb_gro_offset(skb) - off;

        list_for_each_entry(p, head, list) {
                const struct ipv6hdr *iph2;
                __be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */

                if (!NAPI_GRO_CB(p)->same_flow)
                        continue;

                iph2 = (struct ipv6hdr *)(p->data + off);
                first_word = *(__be32 *)iph ^ *(__be32 *)iph2;

                /* All fields must match except length and Traffic Class.
                 * XXX skbs on the gro_list have all been parsed and pulled
                 * already so we don't need to compare nlen
                 * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
                 * memcmp() alone below is sufficient, right?
                 */
                 if ((first_word & htonl(0xF00FFFFF)) ||
                     !ipv6_addr_equal(&iph->saddr, &iph2->saddr) ||
                     !ipv6_addr_equal(&iph->daddr, &iph2->daddr) ||
                     iph->nexthdr != iph2->nexthdr) {
not_same_flow:
                        NAPI_GRO_CB(p)->same_flow = 0;
                        continue;
                }
                if (unlikely(nlen > sizeof(struct ipv6hdr))) {
                        if (memcmp(iph + 1, iph2 + 1,
                                   nlen - sizeof(struct ipv6hdr)))
                                goto not_same_flow;
                }
        }

        NAPI_GRO_CB(skb)->flush |= flush;

        skb_gro_postpull_rcsum(skb, iph, nlen);

        pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive,
                                         ops->callbacks.gro_receive, head, skb);

out:
        skb_gro_flush_final(skb, pp, flush);

        return pp;
}

static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
                                              struct sk_buff *skb)
{
        /* Common GRO receive for SIT and IP6IP6 */

        if (NAPI_GRO_CB(skb)->encap_mark) {
                NAPI_GRO_CB(skb)->flush = 1;
                return NULL;
        }

        NAPI_GRO_CB(skb)->encap_mark = 1;

        return ipv6_gro_receive(head, skb);
}

static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
                                          struct sk_buff *skb)
{
        /* Common GRO receive for SIT and IP6IP6 */

        if (NAPI_GRO_CB(skb)->encap_mark) {
                NAPI_GRO_CB(skb)->flush = 1;
                return NULL;
        }

        NAPI_GRO_CB(skb)->encap_mark = 1;

        return inet_gro_receive(head, skb);
}

INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
{
        const struct net_offload *ops;
        struct ipv6hdr *iph;
        int err = -ENOSYS;
        u32 payload_len;

        if (skb->encapsulation) {
                skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
                skb_set_inner_network_header(skb, nhoff);
        }

        payload_len = skb->len - nhoff - sizeof(*iph);
        if (unlikely(payload_len > IPV6_MAXPLEN)) {
                struct hop_jumbo_hdr *hop_jumbo;
                int hoplen = sizeof(*hop_jumbo);

                /* Move network header left */
                memmove(skb_mac_header(skb) - hoplen, skb_mac_header(skb),
                        skb->transport_header - skb->mac_header);
                skb->data -= hoplen;
                skb->len += hoplen;
                skb->mac_header -= hoplen;
                skb->network_header -= hoplen;
                iph = (struct ipv6hdr *)(skb->data + nhoff);
                hop_jumbo = (struct hop_jumbo_hdr *)(iph + 1);

                /* Build hop-by-hop options */
                hop_jumbo->nexthdr = iph->nexthdr;
                hop_jumbo->hdrlen = 0;
                hop_jumbo->tlv_type = IPV6_TLV_JUMBO;
                hop_jumbo->tlv_len = 4;
                hop_jumbo->jumbo_payload_len = htonl(payload_len + hoplen);

                iph->nexthdr = NEXTHDR_HOP;
                iph->payload_len = 0;
        } else {
                iph = (struct ipv6hdr *)(skb->data + nhoff);
                iph->payload_len = htons(payload_len);
        }

        nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
        if (WARN_ON(!ops || !ops->callbacks.gro_complete))
                goto out;

        err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete,
                               udp6_gro_complete, skb, nhoff);

out:
        return err;
}

static int sit_gro_complete(struct sk_buff *skb, int nhoff)
{
        skb->encapsulation = 1;
        skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
        return ipv6_gro_complete(skb, nhoff);
}

static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)
{
        skb->encapsulation = 1;
        skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
        return ipv6_gro_complete(skb, nhoff);
}

static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)
{
        skb->encapsulation = 1;
        skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
        return inet_gro_complete(skb, nhoff);
}


static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
                                       netdev_features_t features)
{
        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
                return ERR_PTR(-EINVAL);

        return ipv6_gso_segment(skb, features);
}

static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,
                                          netdev_features_t features)
{
        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
                return ERR_PTR(-EINVAL);

        return inet_gso_segment(skb, features);
}

static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,
                                          netdev_features_t features)
{
        if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
                return ERR_PTR(-EINVAL);

        return ipv6_gso_segment(skb, features);
}

static const struct net_offload sit_offload = {
        .callbacks = {
                .gso_segment        = sit_gso_segment,
                .gro_receive    = sit_ip6ip6_gro_receive,
                .gro_complete   = sit_gro_complete,
        },
};

static const struct net_offload ip4ip6_offload = {
        .callbacks = {
                .gso_segment        = ip4ip6_gso_segment,
                .gro_receive    = ip4ip6_gro_receive,
                .gro_complete   = ip4ip6_gro_complete,
        },
};

static const struct net_offload ip6ip6_offload = {
        .callbacks = {
                .gso_segment        = ip6ip6_gso_segment,
                .gro_receive    = sit_ip6ip6_gro_receive,
                .gro_complete   = ip6ip6_gro_complete,
        },
};
static int __init ipv6_offload_init(void)
{

        if (tcpv6_offload_init() < 0)
                pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
        if (ipv6_exthdrs_offload_init() < 0)
                pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);

        net_hotdata.ipv6_packet_offload = (struct packet_offload) {
                .type = cpu_to_be16(ETH_P_IPV6),
                .callbacks = {
                        .gso_segment = ipv6_gso_segment,
                        .gro_receive = ipv6_gro_receive,
                        .gro_complete = ipv6_gro_complete,
                },
        };
        dev_add_offload(&net_hotdata.ipv6_packet_offload);

        inet_add_offload(&sit_offload, IPPROTO_IPV6);
        inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
        inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP);

        return 0;
}

fs_initcall(ipv6_offload_init);


















































   11 




  133 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_MBCACHE_H
#define _LINUX_MBCACHE_H

#include <linux/hash.h>
#include <linux/list_bl.h>
#include <linux/list.h>
#include <linux/atomic.h>
#include <linux/fs.h>

struct mb_cache;

/* Cache entry flags */
enum {
        MBE_REFERENCED_B = 0,
        MBE_REUSABLE_B
};

struct mb_cache_entry {
        /* List of entries in cache - protected by cache->c_list_lock */
        struct list_head        e_list;
        /*
         * Hash table list - protected by hash chain bitlock. The entry is
         * guaranteed to be hashed while e_refcnt > 0.
         */
        struct hlist_bl_node        e_hash_list;
        /*
         * Entry refcount. Once it reaches zero, entry is unhashed and freed.
         * While refcount > 0, the entry is guaranteed to stay in the hash and
         * e.g. mb_cache_entry_try_delete() will fail.
         */
        atomic_t                e_refcnt;
        /* Key in hash - stable during lifetime of the entry */
        u32                        e_key;
        unsigned long                e_flags;
        /* User provided value - stable during lifetime of the entry */
        u64                        e_value;
};

struct mb_cache *mb_cache_create(int bucket_bits);
void mb_cache_destroy(struct mb_cache *cache);

int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key,
                          u64 value, bool reusable);
void __mb_cache_entry_free(struct mb_cache *cache,
                           struct mb_cache_entry *entry);
void mb_cache_entry_wait_unused(struct mb_cache_entry *entry);
static inline void mb_cache_entry_put(struct mb_cache *cache,
                                      struct mb_cache_entry *entry)
{
        unsigned int cnt = atomic_dec_return(&entry->e_refcnt);

        if (cnt > 0) {
                if (cnt <= 2)
                        wake_up_var(&entry->e_refcnt);
                return;
        }
        __mb_cache_entry_free(cache, entry);
}

struct mb_cache_entry *mb_cache_entry_delete_or_get(struct mb_cache *cache,
                                                    u32 key, u64 value);
struct mb_cache_entry *mb_cache_entry_get(struct mb_cache *cache, u32 key,
                                          u64 value);
struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache,
                                                 u32 key);
struct mb_cache_entry *mb_cache_entry_find_next(struct mb_cache *cache,
                                                struct mb_cache_entry *entry);
void mb_cache_entry_touch(struct mb_cache *cache,
                          struct mb_cache_entry *entry);

#endif        /* _LINUX_MBCACHE_H */









































































































































































































































































  128 

    2 

  130 
  127 













   27 



   26 












  205 
  205 











  130 



  129 
   32 













  124 


   26 
  125 













  202 

    2 

  200 














  221 

    2 

  224 
    7 














  194 
  195 













    2 


    2 
















   84 














   82 















    2 


































































    5 


    5 






























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
/*
 *  PCM Interface - misc routines
 *  Copyright (c) 1998 by Jaroslav Kysela <perex@perex.cz>
 *
 *
 *   This library is free software; you can redistribute it and/or modify
 *   it under the terms of the GNU Library General Public License as
 *   published by the Free Software Foundation; either version 2 of
 *   the License, or (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU Library General Public License for more details.
 *
 *   You should have received a copy of the GNU Library General Public
 *   License along with this library; if not, write to the Free Software
 *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 *
 */
  
#include <linux/time.h>
#include <linux/export.h>
#include <sound/core.h>
#include <sound/pcm.h>

#include "pcm_local.h"

#define SND_PCM_FORMAT_UNKNOWN (-1)

/* NOTE: "signed" prefix must be given below since the default char is
 *       unsigned on some architectures!
 */
struct pcm_format_data {
        unsigned char width;        /* bit width */
        unsigned char phys;        /* physical bit width */
        signed char le;        /* 0 = big-endian, 1 = little-endian, -1 = others */
        signed char signd;        /* 0 = unsigned, 1 = signed, -1 = others */
        unsigned char silence[8];        /* silence data to fill */
};

/* we do lots of calculations on snd_pcm_format_t; shut up sparse */
#define INT        __force int

static bool valid_format(snd_pcm_format_t format)
{
        return (INT)format >= 0 && (INT)format <= (INT)SNDRV_PCM_FORMAT_LAST;
}

static const struct pcm_format_data pcm_formats[(INT)SNDRV_PCM_FORMAT_LAST+1] = {
        [SNDRV_PCM_FORMAT_S8] = {
                .width = 8, .phys = 8, .le = -1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U8] = {
                .width = 8, .phys = 8, .le = -1, .signd = 0,
                .silence = { 0x80 },
        },
        [SNDRV_PCM_FORMAT_S16_LE] = {
                .width = 16, .phys = 16, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S16_BE] = {
                .width = 16, .phys = 16, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U16_LE] = {
                .width = 16, .phys = 16, .le = 1, .signd = 0,
                .silence = { 0x00, 0x80 },
        },
        [SNDRV_PCM_FORMAT_U16_BE] = {
                .width = 16, .phys = 16, .le = 0, .signd = 0,
                .silence = { 0x80, 0x00 },
        },
        [SNDRV_PCM_FORMAT_S24_LE] = {
                .width = 24, .phys = 32, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S24_BE] = {
                .width = 24, .phys = 32, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U24_LE] = {
                .width = 24, .phys = 32, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x80 },
        },
        [SNDRV_PCM_FORMAT_U24_BE] = {
                .width = 24, .phys = 32, .le = 0, .signd = 0,
                .silence = { 0x00, 0x80, 0x00, 0x00 },
        },
        [SNDRV_PCM_FORMAT_S32_LE] = {
                .width = 32, .phys = 32, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S32_BE] = {
                .width = 32, .phys = 32, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U32_LE] = {
                .width = 32, .phys = 32, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x00, 0x80 },
        },
        [SNDRV_PCM_FORMAT_U32_BE] = {
                .width = 32, .phys = 32, .le = 0, .signd = 0,
                .silence = { 0x80, 0x00, 0x00, 0x00 },
        },
        [SNDRV_PCM_FORMAT_FLOAT_LE] = {
                .width = 32, .phys = 32, .le = 1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_FLOAT_BE] = {
                .width = 32, .phys = 32, .le = 0, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_FLOAT64_LE] = {
                .width = 64, .phys = 64, .le = 1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_FLOAT64_BE] = {
                .width = 64, .phys = 64, .le = 0, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_IEC958_SUBFRAME_LE] = {
                .width = 32, .phys = 32, .le = 1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_IEC958_SUBFRAME_BE] = {
                .width = 32, .phys = 32, .le = 0, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_MU_LAW] = {
                .width = 8, .phys = 8, .le = -1, .signd = -1,
                .silence = { 0x7f },
        },
        [SNDRV_PCM_FORMAT_A_LAW] = {
                .width = 8, .phys = 8, .le = -1, .signd = -1,
                .silence = { 0x55 },
        },
        [SNDRV_PCM_FORMAT_IMA_ADPCM] = {
                .width = 4, .phys = 4, .le = -1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_G723_24] = {
                .width = 3, .phys = 3, .le = -1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_G723_40] = {
                .width = 5, .phys = 5, .le = -1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_DSD_U8] = {
                .width = 8, .phys = 8, .le = 1, .signd = 0,
                .silence = { 0x69 },
        },
        [SNDRV_PCM_FORMAT_DSD_U16_LE] = {
                .width = 16, .phys = 16, .le = 1, .signd = 0,
                .silence = { 0x69, 0x69 },
        },
        [SNDRV_PCM_FORMAT_DSD_U32_LE] = {
                .width = 32, .phys = 32, .le = 1, .signd = 0,
                .silence = { 0x69, 0x69, 0x69, 0x69 },
        },
        [SNDRV_PCM_FORMAT_DSD_U16_BE] = {
                .width = 16, .phys = 16, .le = 0, .signd = 0,
                .silence = { 0x69, 0x69 },
        },
        [SNDRV_PCM_FORMAT_DSD_U32_BE] = {
                .width = 32, .phys = 32, .le = 0, .signd = 0,
                .silence = { 0x69, 0x69, 0x69, 0x69 },
        },
        /* FIXME: the following two formats are not defined properly yet */
        [SNDRV_PCM_FORMAT_MPEG] = {
                .le = -1, .signd = -1,
        },
        [SNDRV_PCM_FORMAT_GSM] = {
                .le = -1, .signd = -1,
        },
        [SNDRV_PCM_FORMAT_S20_LE] = {
                .width = 20, .phys = 32, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S20_BE] = {
                .width = 20, .phys = 32, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U20_LE] = {
                .width = 20, .phys = 32, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x08, 0x00 },
        },
        [SNDRV_PCM_FORMAT_U20_BE] = {
                .width = 20, .phys = 32, .le = 0, .signd = 0,
                .silence = { 0x00, 0x08, 0x00, 0x00 },
        },
        /* FIXME: the following format is not defined properly yet */
        [SNDRV_PCM_FORMAT_SPECIAL] = {
                .le = -1, .signd = -1,
        },
        [SNDRV_PCM_FORMAT_S24_3LE] = {
                .width = 24, .phys = 24, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S24_3BE] = {
                .width = 24, .phys = 24, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U24_3LE] = {
                .width = 24, .phys = 24, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x80 },
        },
        [SNDRV_PCM_FORMAT_U24_3BE] = {
                .width = 24, .phys = 24, .le = 0, .signd = 0,
                .silence = { 0x80, 0x00, 0x00 },
        },
        [SNDRV_PCM_FORMAT_S20_3LE] = {
                .width = 20, .phys = 24, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S20_3BE] = {
                .width = 20, .phys = 24, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U20_3LE] = {
                .width = 20, .phys = 24, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x08 },
        },
        [SNDRV_PCM_FORMAT_U20_3BE] = {
                .width = 20, .phys = 24, .le = 0, .signd = 0,
                .silence = { 0x08, 0x00, 0x00 },
        },
        [SNDRV_PCM_FORMAT_S18_3LE] = {
                .width = 18, .phys = 24, .le = 1, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_S18_3BE] = {
                .width = 18, .phys = 24, .le = 0, .signd = 1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_U18_3LE] = {
                .width = 18, .phys = 24, .le = 1, .signd = 0,
                .silence = { 0x00, 0x00, 0x02 },
        },
        [SNDRV_PCM_FORMAT_U18_3BE] = {
                .width = 18, .phys = 24, .le = 0, .signd = 0,
                .silence = { 0x02, 0x00, 0x00 },
        },
        [SNDRV_PCM_FORMAT_G723_24_1B] = {
                .width = 3, .phys = 8, .le = -1, .signd = -1,
                .silence = {},
        },
        [SNDRV_PCM_FORMAT_G723_40_1B] = {
                .width = 5, .phys = 8, .le = -1, .signd = -1,
                .silence = {},
        },
};


/**
 * snd_pcm_format_signed - Check the PCM format is signed linear
 * @format: the format to check
 *
 * Return: 1 if the given PCM format is signed linear, 0 if unsigned
 * linear, and a negative error code for non-linear formats.
 */
int snd_pcm_format_signed(snd_pcm_format_t format)
{
        int val;
        if (!valid_format(format))
                return -EINVAL;
        val = pcm_formats[(INT)format].signd;
        if (val < 0)
                return -EINVAL;
        return val;
}
EXPORT_SYMBOL(snd_pcm_format_signed);

/**
 * snd_pcm_format_unsigned - Check the PCM format is unsigned linear
 * @format: the format to check
 *
 * Return: 1 if the given PCM format is unsigned linear, 0 if signed
 * linear, and a negative error code for non-linear formats.
 */
int snd_pcm_format_unsigned(snd_pcm_format_t format)
{
        int val;

        val = snd_pcm_format_signed(format);
        if (val < 0)
                return val;
        return !val;
}
EXPORT_SYMBOL(snd_pcm_format_unsigned);

/**
 * snd_pcm_format_linear - Check the PCM format is linear
 * @format: the format to check
 *
 * Return: 1 if the given PCM format is linear, 0 if not.
 */
int snd_pcm_format_linear(snd_pcm_format_t format)
{
        return snd_pcm_format_signed(format) >= 0;
}
EXPORT_SYMBOL(snd_pcm_format_linear);

/**
 * snd_pcm_format_little_endian - Check the PCM format is little-endian
 * @format: the format to check
 *
 * Return: 1 if the given PCM format is little-endian, 0 if
 * big-endian, or a negative error code if endian not specified.
 */
int snd_pcm_format_little_endian(snd_pcm_format_t format)
{
        int val;
        if (!valid_format(format))
                return -EINVAL;
        val = pcm_formats[(INT)format].le;
        if (val < 0)
                return -EINVAL;
        return val;
}
EXPORT_SYMBOL(snd_pcm_format_little_endian);

/**
 * snd_pcm_format_big_endian - Check the PCM format is big-endian
 * @format: the format to check
 *
 * Return: 1 if the given PCM format is big-endian, 0 if
 * little-endian, or a negative error code if endian not specified.
 */
int snd_pcm_format_big_endian(snd_pcm_format_t format)
{
        int val;

        val = snd_pcm_format_little_endian(format);
        if (val < 0)
                return val;
        return !val;
}
EXPORT_SYMBOL(snd_pcm_format_big_endian);

/**
 * snd_pcm_format_width - return the bit-width of the format
 * @format: the format to check
 *
 * Return: The bit-width of the format, or a negative error code
 * if unknown format.
 */
int snd_pcm_format_width(snd_pcm_format_t format)
{
        int val;
        if (!valid_format(format))
                return -EINVAL;
        val = pcm_formats[(INT)format].width;
        if (!val)
                return -EINVAL;
        return val;
}
EXPORT_SYMBOL(snd_pcm_format_width);

/**
 * snd_pcm_format_physical_width - return the physical bit-width of the format
 * @format: the format to check
 *
 * Return: The physical bit-width of the format, or a negative error code
 * if unknown format.
 */
int snd_pcm_format_physical_width(snd_pcm_format_t format)
{
        int val;
        if (!valid_format(format))
                return -EINVAL;
        val = pcm_formats[(INT)format].phys;
        if (!val)
                return -EINVAL;
        return val;
}
EXPORT_SYMBOL(snd_pcm_format_physical_width);

/**
 * snd_pcm_format_size - return the byte size of samples on the given format
 * @format: the format to check
 * @samples: sampling rate
 *
 * Return: The byte size of the given samples for the format, or a
 * negative error code if unknown format.
 */
ssize_t snd_pcm_format_size(snd_pcm_format_t format, size_t samples)
{
        int phys_width = snd_pcm_format_physical_width(format);
        if (phys_width < 0)
                return -EINVAL;
        return samples * phys_width / 8;
}
EXPORT_SYMBOL(snd_pcm_format_size);

/**
 * snd_pcm_format_silence_64 - return the silent data in 8 bytes array
 * @format: the format to check
 *
 * Return: The format pattern to fill or %NULL if error.
 */
const unsigned char *snd_pcm_format_silence_64(snd_pcm_format_t format)
{
        if (!valid_format(format))
                return NULL;
        if (! pcm_formats[(INT)format].phys)
                return NULL;
        return pcm_formats[(INT)format].silence;
}
EXPORT_SYMBOL(snd_pcm_format_silence_64);

/**
 * snd_pcm_format_set_silence - set the silence data on the buffer
 * @format: the PCM format
 * @data: the buffer pointer
 * @samples: the number of samples to set silence
 *
 * Sets the silence data on the buffer for the given samples.
 *
 * Return: Zero if successful, or a negative error code on failure.
 */
int snd_pcm_format_set_silence(snd_pcm_format_t format, void *data, unsigned int samples)
{
        int width;
        unsigned char *dst;
        const unsigned char *pat;

        if (!valid_format(format))
                return -EINVAL;
        if (samples == 0)
                return 0;
        width = pcm_formats[(INT)format].phys; /* physical width */
        if (!width)
                return -EINVAL;
        pat = pcm_formats[(INT)format].silence;
        /* signed or 1 byte data */
        if (pcm_formats[(INT)format].signd == 1 || width <= 8) {
                unsigned int bytes = samples * width / 8;
                memset(data, *pat, bytes);
                return 0;
        }
        /* non-zero samples, fill using a loop */
        width /= 8;
        dst = data;
#if 0
        while (samples--) {
                memcpy(dst, pat, width);
                dst += width;
        }
#else
        /* a bit optimization for constant width */
        switch (width) {
        case 2:
                while (samples--) {
                        memcpy(dst, pat, 2);
                        dst += 2;
                }
                break;
        case 3:
                while (samples--) {
                        memcpy(dst, pat, 3);
                        dst += 3;
                }
                break;
        case 4:
                while (samples--) {
                        memcpy(dst, pat, 4);
                        dst += 4;
                }
                break;
        case 8:
                while (samples--) {
                        memcpy(dst, pat, 8);
                        dst += 8;
                }
                break;
        }
#endif
        return 0;
}
EXPORT_SYMBOL(snd_pcm_format_set_silence);

/**
 * snd_pcm_hw_limit_rates - determine rate_min/rate_max fields
 * @hw: the pcm hw instance
 *
 * Determines the rate_min and rate_max fields from the rates bits of
 * the given hw.
 *
 * Return: Zero if successful.
 */
int snd_pcm_hw_limit_rates(struct snd_pcm_hardware *hw)
{
        int i;
        unsigned int rmin, rmax;

        rmin = UINT_MAX;
        rmax = 0;
        for (i = 0; i < (int)snd_pcm_known_rates.count; i++) {
                if (hw->rates & (1 << i)) {
                        rmin = min(rmin, snd_pcm_known_rates.list[i]);
                        rmax = max(rmax, snd_pcm_known_rates.list[i]);
                }
        }
        if (rmin > rmax)
                return -EINVAL;
        hw->rate_min = rmin;
        hw->rate_max = rmax;
        return 0;
}
EXPORT_SYMBOL(snd_pcm_hw_limit_rates);

/**
 * snd_pcm_rate_to_rate_bit - converts sample rate to SNDRV_PCM_RATE_xxx bit
 * @rate: the sample rate to convert
 *
 * Return: The SNDRV_PCM_RATE_xxx flag that corresponds to the given rate, or
 * SNDRV_PCM_RATE_KNOT for an unknown rate.
 */
unsigned int snd_pcm_rate_to_rate_bit(unsigned int rate)
{
        unsigned int i;

        for (i = 0; i < snd_pcm_known_rates.count; i++)
                if (snd_pcm_known_rates.list[i] == rate)
                        return 1u << i;
        return SNDRV_PCM_RATE_KNOT;
}
EXPORT_SYMBOL(snd_pcm_rate_to_rate_bit);

/**
 * snd_pcm_rate_bit_to_rate - converts SNDRV_PCM_RATE_xxx bit to sample rate
 * @rate_bit: the rate bit to convert
 *
 * Return: The sample rate that corresponds to the given SNDRV_PCM_RATE_xxx flag
 * or 0 for an unknown rate bit.
 */
unsigned int snd_pcm_rate_bit_to_rate(unsigned int rate_bit)
{
        unsigned int i;

        for (i = 0; i < snd_pcm_known_rates.count; i++)
                if ((1u << i) == rate_bit)
                        return snd_pcm_known_rates.list[i];
        return 0;
}
EXPORT_SYMBOL(snd_pcm_rate_bit_to_rate);

static unsigned int snd_pcm_rate_mask_sanitize(unsigned int rates)
{
        if (rates & SNDRV_PCM_RATE_CONTINUOUS)
                return SNDRV_PCM_RATE_CONTINUOUS;
        else if (rates & SNDRV_PCM_RATE_KNOT)
                return SNDRV_PCM_RATE_KNOT;
        return rates;
}

/**
 * snd_pcm_rate_mask_intersect - computes the intersection between two rate masks
 * @rates_a: The first rate mask
 * @rates_b: The second rate mask
 *
 * This function computes the rates that are supported by both rate masks passed
 * to the function. It will take care of the special handling of
 * SNDRV_PCM_RATE_CONTINUOUS and SNDRV_PCM_RATE_KNOT.
 *
 * Return: A rate mask containing the rates that are supported by both rates_a
 * and rates_b.
 */
unsigned int snd_pcm_rate_mask_intersect(unsigned int rates_a,
        unsigned int rates_b)
{
        rates_a = snd_pcm_rate_mask_sanitize(rates_a);
        rates_b = snd_pcm_rate_mask_sanitize(rates_b);

        if (rates_a & SNDRV_PCM_RATE_CONTINUOUS)
                return rates_b;
        else if (rates_b & SNDRV_PCM_RATE_CONTINUOUS)
                return rates_a;
        else if (rates_a & SNDRV_PCM_RATE_KNOT)
                return rates_b;
        else if (rates_b & SNDRV_PCM_RATE_KNOT)
                return rates_a;
        return rates_a & rates_b;
}
EXPORT_SYMBOL_GPL(snd_pcm_rate_mask_intersect);

















































































































































































































































































































































































































    1 














































    1 














    1 































































































































































































    5 




























    1 





    4 



    3 





    3 



    1 



    3 






    1 





    1 
























    1 



    1 



    1 













    1 

















    1 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
// SPDX-License-Identifier: GPL-2.0
/* Driver for Microtek Scanmaker X6 USB scanner, and possibly others.
 *
 * (C) Copyright 2000 John Fremlin <vii@penguinpowered.com>
 * (C) Copyright 2000 Oliver Neukum <Oliver.Neukum@lrz.uni-muenchen.de>
 *
 * Parts shamelessly stolen from usb-storage and copyright by their
 * authors. Thanks to Matt Dharm for giving us permission!
 *
 * This driver implements a SCSI host controller driver and a USB
 * device driver. To avoid confusion, all the USB related stuff is
 * prefixed by mts_usb_ and all the SCSI stuff by mts_scsi_.
 *
 * Microtek (www.microtek.com) did not release the specifications for
 * their USB protocol to us, so we had to reverse engineer them. We
 * don't know for which models they are valid.
 *
 * The X6 USB has three bulk endpoints, one output (0x1) down which
 * commands and outgoing data are sent, and two input: 0x82 from which
 * normal data is read from the scanner (in packets of maximum 32
 * bytes) and from which the status byte is read, and 0x83 from which
 * the results of a scan (or preview) are read in up to 64 * 1024 byte
 * chunks by the Windows driver. We don't know how much it is possible
 * to read at a time from 0x83.
 *
 * It seems possible to read (with URB transfers) everything from 0x82
 * in one go, without bothering to read in 32 byte chunks.
 *
 * There seems to be an optimisation of a further READ implicit if
 * you simply read from 0x83.
 *
 * Guessed protocol:
 *
 *        Send raw SCSI command to EP 0x1
 *
 *        If there is data to receive:
 *                If the command was READ datatype=image:
 *                        Read a lot of data from EP 0x83
 *                Else:
 *                        Read data from EP 0x82
 *        Else:
 *                If there is data to transmit:
 *                        Write it to EP 0x1
 *
 *        Read status byte from EP 0x82
 *
 * References:
 *
 * The SCSI command set for the scanner is available from
 *        ftp://ftp.microtek.com/microtek/devpack/
 *
 * Microtek NV sent us a more up to date version of the document. If
 * you want it, just send mail.
 *
 * Status:
 *
 *        Untested with multiple scanners.
 *        Untested on SMP.
 *        Untested on a bigendian machine.
 *
 * History:
 *
 *        20000417 starting history
 *        20000417 fixed load oops
 *        20000417 fixed unload oops
 *        20000419 fixed READ IMAGE detection
 *        20000424 started conversion to use URBs
 *        20000502 handled short transfers as errors
 *        20000513 rename and organisation of functions (john)
 *        20000513 added IDs for all products supported by Windows driver (john)
 *        20000514 Rewrote mts_scsi_queuecommand to use URBs (john)
 *        20000514 Version 0.0.8j
 *      20000514 Fix reporting of non-existent devices to SCSI layer (john)
 *        20000514 Added MTS_DEBUG_INT (john)
 *        20000514 Changed "usb-microtek" to "microtek" for consistency (john)
 *        20000514 Stupid bug fixes (john)
 *        20000514 Version 0.0.9j
 *        20000515 Put transfer context and URB in mts_desc (john)
 *        20000515 Added prelim turn off debugging support (john)
 *        20000515 Version 0.0.10j
 *      20000515 Fixed up URB allocation (clear URB on alloc) (john)
 *      20000515 Version 0.0.11j
 *        20000516 Removed unnecessary spinlock in mts_transfer_context (john)
 *        20000516 Removed unnecessary up on instance lock in mts_remove_nolock (john)
 *        20000516 Implemented (badly) scsi_abort (john)
 *        20000516 Version 0.0.12j
 *      20000517 Hopefully removed mts_remove_nolock quasideadlock (john)
 *      20000517 Added mts_debug_dump to print ll USB info (john)
 *        20000518 Tweaks and documentation updates (john)
 *        20000518 Version 0.0.13j
 *        20000518 Cleaned up abort handling (john)
 *        20000523 Removed scsi_command and various scsi_..._resets (john)
 *        20000523 Added unlink URB on scsi_abort, now OHCI supports it (john)
 *        20000523 Fixed last tiresome compile warning (john)
 *        20000523 Version 0.0.14j (though version 0.1 has come out?)
 *        20000602 Added primitive reset
 *        20000602 Version 0.2.0
 *        20000603 various cosmetic changes
 *        20000603 Version 0.2.1
 *        20000620 minor cosmetic changes
 *        20000620 Version 0.2.2
 *        20000822 Hopefully fixed deadlock in mts_remove_nolock()
 *        20000822 Fixed minor race in mts_transfer_cleanup()
 *        20000822 Fixed deadlock on submission error in queuecommand
 *        20000822 Version 0.2.3
 *        20000913 Reduced module size if debugging is off
 *        20000913 Version 0.2.4
 *      20010210 New abort logic
 *      20010210 Version 0.3.0
 *        20010217 Merged scatter/gather
 *        20010218 Version 0.4.0
 *        20010218 Cosmetic fixes
 *        20010218 Version 0.4.1
 *      20010306 Abort while using scatter/gather
 *      20010306 Version 0.4.2
 *      20010311 Remove all timeouts and tidy up generally (john)
 *        20010320 check return value of scsi_register()
 *        20010320 Version 0.4.3
 *        20010408 Identify version on module load.
 *        20011003 Fix multiple requests
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/errno.h>
#include <linux/random.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/usb.h>
#include <linux/proc_fs.h>
#include <linux/atomic.h>
#include <linux/blkdev.h>

#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi_tcq.h>

#include "microtek.h"

#define DRIVER_AUTHOR "John Fremlin <vii@penguinpowered.com>, Oliver Neukum <Oliver.Neukum@lrz.uni-muenchen.de>"
#define DRIVER_DESC "Microtek Scanmaker X6 USB scanner driver"

/* Should we do debugging? */

//#define MTS_DO_DEBUG

/* USB layer driver interface */

static int mts_usb_probe(struct usb_interface *intf,
                         const struct usb_device_id *id);
static void mts_usb_disconnect(struct usb_interface *intf);

static const struct usb_device_id mts_usb_ids[];

static struct usb_driver mts_usb_driver = {
        .name =                "microtekX6",
        .probe =        mts_usb_probe,
        .disconnect =        mts_usb_disconnect,
        .id_table =        mts_usb_ids,
};


/* Internal driver stuff */

#define MTS_VERSION        "0.4.3"
#define MTS_NAME        "microtek usb (rev " MTS_VERSION "): "

#define MTS_WARNING(x...) \
        printk( KERN_WARNING MTS_NAME x )
#define MTS_ERROR(x...) \
        printk( KERN_ERR MTS_NAME x )
#define MTS_INT_ERROR(x...) \
        MTS_ERROR(x)
#define MTS_MESSAGE(x...) \
        printk( KERN_INFO MTS_NAME x )

#if defined MTS_DO_DEBUG

#define MTS_DEBUG(x...) \
        printk( KERN_DEBUG MTS_NAME x )

#define MTS_DEBUG_GOT_HERE() \
        MTS_DEBUG("got to %s:%d (%s)\n", __FILE__, (int)__LINE__, __func__ )
#define MTS_DEBUG_INT() \
        do { MTS_DEBUG_GOT_HERE(); \
             MTS_DEBUG("transfer = 0x%x context = 0x%x\n",(int)transfer,(int)context ); \
             MTS_DEBUG("status = 0x%x data-length = 0x%x sent = 0x%x\n",transfer->status,(int)context->data_length, (int)transfer->actual_length ); \
             mts_debug_dump(context->instance);\
           } while(0)
#else

#define MTS_NUL_STATEMENT do { } while(0)

#define MTS_DEBUG(x...)        MTS_NUL_STATEMENT
#define MTS_DEBUG_GOT_HERE() MTS_NUL_STATEMENT
#define MTS_DEBUG_INT() MTS_NUL_STATEMENT

#endif



#define MTS_INT_INIT()\
        struct mts_transfer_context* context = (struct mts_transfer_context*)transfer->context; \
        MTS_DEBUG_INT();\

#ifdef MTS_DO_DEBUG

static inline void mts_debug_dump(struct mts_desc* desc) {
        MTS_DEBUG("desc at 0x%x: toggle = %02x%02x\n",
                  (int)desc,
                  (int)desc->usb_dev->toggle[1],(int)desc->usb_dev->toggle[0]
                );
        MTS_DEBUG("ep_out=%x ep_response=%x ep_image=%x\n",
                  usb_sndbulkpipe(desc->usb_dev,desc->ep_out),
                  usb_rcvbulkpipe(desc->usb_dev,desc->ep_response),
                  usb_rcvbulkpipe(desc->usb_dev,desc->ep_image)
                );
}


static inline void mts_show_command(struct scsi_cmnd *srb)
{
        char *what = NULL;

        switch (srb->cmnd[0]) {
        case TEST_UNIT_READY: what = "TEST_UNIT_READY"; break;
        case REZERO_UNIT: what = "REZERO_UNIT"; break;
        case REQUEST_SENSE: what = "REQUEST_SENSE"; break;
        case FORMAT_UNIT: what = "FORMAT_UNIT"; break;
        case READ_BLOCK_LIMITS: what = "READ_BLOCK_LIMITS"; break;
        case REASSIGN_BLOCKS: what = "REASSIGN_BLOCKS"; break;
        case READ_6: what = "READ_6"; break;
        case WRITE_6: what = "WRITE_6"; break;
        case SEEK_6: what = "SEEK_6"; break;
        case READ_REVERSE: what = "READ_REVERSE"; break;
        case WRITE_FILEMARKS: what = "WRITE_FILEMARKS"; break;
        case SPACE: what = "SPACE"; break;
        case INQUIRY: what = "INQUIRY"; break;
        case RECOVER_BUFFERED_DATA: what = "RECOVER_BUFFERED_DATA"; break;
        case MODE_SELECT: what = "MODE_SELECT"; break;
        case RESERVE: what = "RESERVE"; break;
        case RELEASE: what = "RELEASE"; break;
        case COPY: what = "COPY"; break;
        case ERASE: what = "ERASE"; break;
        case MODE_SENSE: what = "MODE_SENSE"; break;
        case START_STOP: what = "START_STOP"; break;
        case RECEIVE_DIAGNOSTIC: what = "RECEIVE_DIAGNOSTIC"; break;
        case SEND_DIAGNOSTIC: what = "SEND_DIAGNOSTIC"; break;
        case ALLOW_MEDIUM_REMOVAL: what = "ALLOW_MEDIUM_REMOVAL"; break;
        case SET_WINDOW: what = "SET_WINDOW"; break;
        case READ_CAPACITY: what = "READ_CAPACITY"; break;
        case READ_10: what = "READ_10"; break;
        case WRITE_10: what = "WRITE_10"; break;
        case SEEK_10: what = "SEEK_10"; break;
        case WRITE_VERIFY: what = "WRITE_VERIFY"; break;
        case VERIFY: what = "VERIFY"; break;
        case SEARCH_HIGH: what = "SEARCH_HIGH"; break;
        case SEARCH_EQUAL: what = "SEARCH_EQUAL"; break;
        case SEARCH_LOW: what = "SEARCH_LOW"; break;
        case SET_LIMITS: what = "SET_LIMITS"; break;
        case READ_POSITION: what = "READ_POSITION"; break;
        case SYNCHRONIZE_CACHE: what = "SYNCHRONIZE_CACHE"; break;
        case LOCK_UNLOCK_CACHE: what = "LOCK_UNLOCK_CACHE"; break;
        case READ_DEFECT_DATA: what = "READ_DEFECT_DATA"; break;
        case MEDIUM_SCAN: what = "MEDIUM_SCAN"; break;
        case COMPARE: what = "COMPARE"; break;
        case COPY_VERIFY: what = "COPY_VERIFY"; break;
        case WRITE_BUFFER: what = "WRITE_BUFFER"; break;
        case READ_BUFFER: what = "READ_BUFFER"; break;
        case UPDATE_BLOCK: what = "UPDATE_BLOCK"; break;
        case READ_LONG: what = "READ_LONG"; break;
        case WRITE_LONG: what = "WRITE_LONG"; break;
        case CHANGE_DEFINITION: what = "CHANGE_DEFINITION"; break;
        case WRITE_SAME: what = "WRITE_SAME"; break;
        case READ_TOC: what = "READ_TOC"; break;
        case LOG_SELECT: what = "LOG_SELECT"; break;
        case LOG_SENSE: what = "LOG_SENSE"; break;
        case MODE_SELECT_10: what = "MODE_SELECT_10"; break;
        case MODE_SENSE_10: what = "MODE_SENSE_10"; break;
        case MOVE_MEDIUM: what = "MOVE_MEDIUM"; break;
        case READ_12: what = "READ_12"; break;
        case WRITE_12: what = "WRITE_12"; break;
        case WRITE_VERIFY_12: what = "WRITE_VERIFY_12"; break;
        case SEARCH_HIGH_12: what = "SEARCH_HIGH_12"; break;
        case SEARCH_EQUAL_12: what = "SEARCH_EQUAL_12"; break;
        case SEARCH_LOW_12: what = "SEARCH_LOW_12"; break;
        case READ_ELEMENT_STATUS: what = "READ_ELEMENT_STATUS"; break;
        case SEND_VOLUME_TAG: what = "SEND_VOLUME_TAG"; break;
        case WRITE_LONG_2: what = "WRITE_LONG_2"; break;
        default:
                MTS_DEBUG("can't decode command\n");
                goto out;
                break;
        }
        MTS_DEBUG( "Command %s (%d bytes)\n", what, srb->cmd_len);

 out:
        MTS_DEBUG( "  %10ph\n", srb->cmnd);
}

#else

static inline void mts_show_command(struct scsi_cmnd * dummy)
{
}

static inline void mts_debug_dump(struct mts_desc* dummy)
{
}

#endif

static inline void mts_urb_abort(struct mts_desc* desc) {
        MTS_DEBUG_GOT_HERE();
        mts_debug_dump(desc);

        usb_kill_urb( desc->urb );
}

static int mts_sdev_init (struct scsi_device *s)
{
        s->inquiry_len = 0x24;
        return 0;
}

static int mts_scsi_abort(struct scsi_cmnd *srb)
{
        struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);

        MTS_DEBUG_GOT_HERE();

        mts_urb_abort(desc);

        return FAILED;
}

static int mts_scsi_host_reset(struct scsi_cmnd *srb)
{
        struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);
        int result;

        MTS_DEBUG_GOT_HERE();
        mts_debug_dump(desc);

        result = usb_lock_device_for_reset(desc->usb_dev, desc->usb_intf);
        if (result == 0) {
                result = usb_reset_device(desc->usb_dev);
                usb_unlock_device(desc->usb_dev);
        }
        return result ? FAILED : SUCCESS;
}

static int
mts_scsi_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *srb);

static void mts_transfer_cleanup( struct urb *transfer );
static void mts_do_sg(struct urb * transfer);

static inline
void mts_int_submit_urb (struct urb* transfer,
                        int pipe,
                        void* data,
                        unsigned length,
                        usb_complete_t callback )
/* Interrupt context! */

/* Holding transfer->context->lock! */
{
        int res;

        MTS_INT_INIT();

        usb_fill_bulk_urb(transfer,
                      context->instance->usb_dev,
                      pipe,
                      data,
                      length,
                      callback,
                      context
                );

        res = usb_submit_urb( transfer, GFP_ATOMIC );
        if ( unlikely(res) ) {
                MTS_INT_ERROR( "could not submit URB! Error was %d\n",(int)res );
                set_host_byte(context->srb, DID_ERROR);
                mts_transfer_cleanup(transfer);
        }
}


static void mts_transfer_cleanup( struct urb *transfer )
/* Interrupt context! */
{
        MTS_INT_INIT();

        if ( likely(context->final_callback != NULL) )
                context->final_callback(context->srb);
}

static void mts_transfer_done( struct urb *transfer )
{
        MTS_INT_INIT();

        context->srb->result &= MTS_SCSI_ERR_MASK;
        context->srb->result |= (unsigned)(*context->scsi_status)<<1;

        mts_transfer_cleanup(transfer);
}


static void mts_get_status( struct urb *transfer )
/* Interrupt context! */
{
        MTS_INT_INIT();

        mts_int_submit_urb(transfer,
                           usb_rcvbulkpipe(context->instance->usb_dev,
                                           context->instance->ep_response),
                           context->scsi_status,
                           1,
                           mts_transfer_done );
}

static void mts_data_done( struct urb* transfer )
/* Interrupt context! */
{
        int status = transfer->status;
        MTS_INT_INIT();

        if ( context->data_length != transfer->actual_length ) {
                scsi_set_resid(context->srb, context->data_length -
                               transfer->actual_length);
        } else if ( unlikely(status) ) {
                set_host_byte(context->srb, (status == -ENOENT ? DID_ABORT : DID_ERROR));
        }

        mts_get_status(transfer);
}


static void mts_command_done( struct urb *transfer )
/* Interrupt context! */
{
        int status = transfer->status;
        MTS_INT_INIT();

        if ( unlikely(status) ) {
                if (status == -ENOENT) {
                        /* We are being killed */
                        MTS_DEBUG_GOT_HERE();
                        set_host_byte(context->srb, DID_ABORT);
                } else {
                        /* A genuine error has occurred */
                        MTS_DEBUG_GOT_HERE();

                        set_host_byte(context->srb, DID_ERROR);
                }
                mts_transfer_cleanup(transfer);

                return;
        }

        if (context->srb->cmnd[0] == REQUEST_SENSE) {
                mts_int_submit_urb(transfer,
                                   context->data_pipe,
                                   context->srb->sense_buffer,
                                   context->data_length,
                                   mts_data_done);
        } else { if ( context->data ) {
                        mts_int_submit_urb(transfer,
                                           context->data_pipe,
                                           context->data,
                                           context->data_length,
                                           scsi_sg_count(context->srb) > 1 ?
                                                   mts_do_sg : mts_data_done);
                } else {
                        mts_get_status(transfer);
                }
        }
}

static void mts_do_sg (struct urb* transfer)
{
        int status = transfer->status;
        MTS_INT_INIT();

        MTS_DEBUG("Processing fragment %d of %d\n", context->fragment,
                                                  scsi_sg_count(context->srb));

        if (unlikely(status)) {
                set_host_byte(context->srb, (status == -ENOENT ? DID_ABORT : DID_ERROR));
                mts_transfer_cleanup(transfer);
        }

        context->curr_sg = sg_next(context->curr_sg);
        mts_int_submit_urb(transfer,
                           context->data_pipe,
                           sg_virt(context->curr_sg),
                           context->curr_sg->length,
                           sg_is_last(context->curr_sg) ?
                           mts_data_done : mts_do_sg);
}

static const u8 mts_read_image_sig[] = { 0x28, 00, 00, 00 };
static const u8 mts_read_image_sig_len = 4;
static const unsigned char mts_direction[256/8] = {
        0x28, 0x81, 0x14, 0x14, 0x20, 0x01, 0x90, 0x77,
        0x0C, 0x20, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};


#define MTS_DIRECTION_IS_IN(x) ((mts_direction[x>>3] >> (x & 7)) & 1)

static void
mts_build_transfer_context(struct scsi_cmnd *srb, struct mts_desc* desc)
{
        int pipe;

        MTS_DEBUG_GOT_HERE();

        desc->context.instance = desc;
        desc->context.srb = srb;

        if (!scsi_bufflen(srb)) {
                desc->context.data = NULL;
                desc->context.data_length = 0;
                return;
        } else {
                desc->context.curr_sg = scsi_sglist(srb);
                desc->context.data = sg_virt(desc->context.curr_sg);
                desc->context.data_length = desc->context.curr_sg->length;
        }


        /* can't rely on srb->sc_data_direction */

        /* Brutally ripped from usb-storage */

        if ( !memcmp( srb->cmnd, mts_read_image_sig, mts_read_image_sig_len )
) {                 pipe = usb_rcvbulkpipe(desc->usb_dev,desc->ep_image);
                MTS_DEBUG( "transferring from desc->ep_image == %d\n",
                           (int)desc->ep_image );
        } else if ( MTS_DIRECTION_IS_IN(srb->cmnd[0]) ) {
                        pipe = usb_rcvbulkpipe(desc->usb_dev,desc->ep_response);
                        MTS_DEBUG( "transferring from desc->ep_response == %d\n",
                                   (int)desc->ep_response);
        } else {
                MTS_DEBUG("transferring to desc->ep_out == %d\n",
                          (int)desc->ep_out);
                pipe = usb_sndbulkpipe(desc->usb_dev,desc->ep_out);
        }
        desc->context.data_pipe = pipe;
}

static int mts_scsi_queuecommand_lck(struct scsi_cmnd *srb)
{
        mts_scsi_cmnd_callback callback = scsi_done;
        struct mts_desc* desc = (struct mts_desc*)(srb->device->host->hostdata[0]);
        int res;

        MTS_DEBUG_GOT_HERE();
        mts_show_command(srb);
        mts_debug_dump(desc);

        if ( srb->device->lun || srb->device->id || srb->device->channel ) {

                MTS_DEBUG("Command to LUN=%d ID=%d CHANNEL=%d from SCSI layer\n",(int)srb->device->lun,(int)srb->device->id, (int)srb->device->channel );

                MTS_DEBUG("this device doesn't exist\n");

                set_host_byte(srb, DID_BAD_TARGET);

                if(likely(callback != NULL))
                        callback(srb);

                goto out;
        }

        
        usb_fill_bulk_urb(desc->urb,
                      desc->usb_dev,
                      usb_sndbulkpipe(desc->usb_dev,desc->ep_out),
                      srb->cmnd,
                      srb->cmd_len,
                      mts_command_done,
                      &desc->context
                      );


        mts_build_transfer_context( srb, desc );
        desc->context.final_callback = callback;
        
        /* here we need ATOMIC as we are called with the iolock */
        res=usb_submit_urb(desc->urb, GFP_ATOMIC);

        if(unlikely(res)){
                MTS_ERROR("error %d submitting URB\n",(int)res);
                set_host_byte(srb, DID_ERROR);

                if(likely(callback != NULL))
                        callback(srb);

        }
out:
        return 0;
}

static DEF_SCSI_QCMD(mts_scsi_queuecommand)

static const struct scsi_host_template mts_scsi_host_template = {
        .module                        = THIS_MODULE,
        .name                        = "microtekX6",
        .proc_name                = "microtekX6",
        .queuecommand                = mts_scsi_queuecommand,
        .eh_abort_handler        = mts_scsi_abort,
        .eh_host_reset_handler        = mts_scsi_host_reset,
        .sg_tablesize =                SG_ALL,
        .can_queue =                1,
        .this_id =                -1,
        .emulated =                1,
        .dma_alignment =        511,
        .sdev_init =                mts_sdev_init,
        .max_sectors=                256, /* 128 K */
};

/* The entries of microtek_table must correspond, line-by-line to
   the entries of mts_supported_products[]. */

static const struct usb_device_id mts_usb_ids[] =
{
        { USB_DEVICE(0x4ce, 0x0300) },
        { USB_DEVICE(0x5da, 0x0094) },
        { USB_DEVICE(0x5da, 0x0099) },
        { USB_DEVICE(0x5da, 0x009a) },
        { USB_DEVICE(0x5da, 0x00a0) },
        { USB_DEVICE(0x5da, 0x00a3) },
        { USB_DEVICE(0x5da, 0x80a3) },
        { USB_DEVICE(0x5da, 0x80ac) },
        { USB_DEVICE(0x5da, 0x00b6) },
        { }                                                /* Terminating entry */
};

MODULE_DEVICE_TABLE (usb, mts_usb_ids);


static int mts_usb_probe(struct usb_interface *intf,
                         const struct usb_device_id *id)
{
        int i;
        int ep_out = -1;
        int ep_in_set[3]; /* this will break if we have more than three endpoints
                           which is why we check */
        int *ep_in_current = ep_in_set;
        int err_retval = -ENOMEM;

        struct mts_desc * new_desc;
        struct usb_device *dev = interface_to_usbdev (intf);

        /* the current altsetting on the interface we're probing */
        struct usb_host_interface *altsetting;

        MTS_DEBUG_GOT_HERE();
        MTS_DEBUG( "usb-device descriptor at %x\n", (int)dev );

        MTS_DEBUG( "product id = 0x%x, vendor id = 0x%x\n",
                   le16_to_cpu(dev->descriptor.idProduct),
                   le16_to_cpu(dev->descriptor.idVendor) );

        MTS_DEBUG_GOT_HERE();

        /* the current altsetting on the interface we're probing */
        altsetting = intf->cur_altsetting;


        /* Check if the config is sane */

        if ( altsetting->desc.bNumEndpoints != MTS_EP_TOTAL ) {
                MTS_WARNING( "expecting %d got %d endpoints! Bailing out.\n",
                             (int)MTS_EP_TOTAL, (int)altsetting->desc.bNumEndpoints );
                return -ENODEV;
        }

        for( i = 0; i < altsetting->desc.bNumEndpoints; i++ ) {
                if ((altsetting->endpoint[i].desc.bmAttributes &
                     USB_ENDPOINT_XFERTYPE_MASK) != USB_ENDPOINT_XFER_BULK) {

                        MTS_WARNING( "can only deal with bulk endpoints; endpoint %d is not bulk.\n",
                             (int)altsetting->endpoint[i].desc.bEndpointAddress );
                } else {
                        if (altsetting->endpoint[i].desc.bEndpointAddress &
                            USB_DIR_IN)
                                *ep_in_current++
                                        = altsetting->endpoint[i].desc.bEndpointAddress &
                                        USB_ENDPOINT_NUMBER_MASK;
                        else {
                                if ( ep_out != -1 ) {
                                        MTS_WARNING( "can only deal with one output endpoints. Bailing out." );
                                        return -ENODEV;
                                }

                                ep_out = altsetting->endpoint[i].desc.bEndpointAddress &
                                        USB_ENDPOINT_NUMBER_MASK;
                        }
                }

        }

        if (ep_in_current != &ep_in_set[2]) {
                MTS_WARNING("couldn't find two input bulk endpoints. Bailing out.\n");
                return -ENODEV;
        }

        if ( ep_out == -1 ) {
                MTS_WARNING( "couldn't find an output bulk endpoint. Bailing out.\n" );
                return -ENODEV;
        }


        new_desc = kzalloc(sizeof(struct mts_desc), GFP_KERNEL);
        if (!new_desc)
                goto out;

        new_desc->urb = usb_alloc_urb(0, GFP_KERNEL);
        if (!new_desc->urb)
                goto out_kfree;

        new_desc->context.scsi_status = kmalloc(1, GFP_KERNEL);
        if (!new_desc->context.scsi_status)
                goto out_free_urb;

        new_desc->usb_dev = dev;
        new_desc->usb_intf = intf;

        /* endpoints */
        new_desc->ep_out = ep_out;
        new_desc->ep_response = ep_in_set[0];
        new_desc->ep_image = ep_in_set[1];

        if ( new_desc->ep_out != MTS_EP_OUT )
                MTS_WARNING( "will this work? Command EP is not usually %d\n",
                             (int)new_desc->ep_out );

        if ( new_desc->ep_response != MTS_EP_RESPONSE )
                MTS_WARNING( "will this work? Response EP is not usually %d\n",
                             (int)new_desc->ep_response );

        if ( new_desc->ep_image != MTS_EP_IMAGE )
                MTS_WARNING( "will this work? Image data EP is not usually %d\n",
                             (int)new_desc->ep_image );

        new_desc->host = scsi_host_alloc(&mts_scsi_host_template,
                        sizeof(new_desc));
        if (!new_desc->host)
                goto out_kfree2;

        new_desc->host->hostdata[0] = (unsigned long)new_desc;
        if (scsi_add_host(new_desc->host, &dev->dev)) {
                err_retval = -EIO;
                goto out_host_put;
        }
        scsi_scan_host(new_desc->host);

        usb_set_intfdata(intf, new_desc);
        return 0;

 out_host_put:
        scsi_host_put(new_desc->host);
 out_kfree2:
        kfree(new_desc->context.scsi_status);
 out_free_urb:
        usb_free_urb(new_desc->urb);
 out_kfree:
        kfree(new_desc);
 out:
        return err_retval;
}

static void mts_usb_disconnect (struct usb_interface *intf)
{
        struct mts_desc *desc = usb_get_intfdata(intf);

        usb_set_intfdata(intf, NULL);

        usb_kill_urb(desc->urb);
        scsi_remove_host(desc->host);

        scsi_host_put(desc->host);
        usb_free_urb(desc->urb);
        kfree(desc->context.scsi_status);
        kfree(desc);
}

module_usb_driver(mts_usb_driver);

MODULE_AUTHOR( DRIVER_AUTHOR );
MODULE_DESCRIPTION( DRIVER_DESC );
MODULE_LICENSE("GPL");






























































































































































































































































































  129 
  129 

























































































































    2 































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (C) 2017-2018 HUAWEI, Inc.
 *             https://www.huawei.com/
 * Copyright (C) 2021, Alibaba Cloud
 */
#ifndef __EROFS_INTERNAL_H
#define __EROFS_INTERNAL_H

#include <linux/fs.h>
#include <linux/dax.h>
#include <linux/dcache.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/bio.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/iomap.h>
#include "erofs_fs.h"

__printf(2, 3) void _erofs_printk(struct super_block *sb, const char *fmt, ...);
#define erofs_err(sb, fmt, ...)        \
        _erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__)
#define erofs_info(sb, fmt, ...) \
        _erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__)

#ifdef CONFIG_EROFS_FS_DEBUG
#define DBG_BUGON               BUG_ON
#else
#define DBG_BUGON(x)            ((void)(x))
#endif        /* !CONFIG_EROFS_FS_DEBUG */

/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */
#define EROFS_SUPER_MAGIC   EROFS_SUPER_MAGIC_V1

typedef u64 erofs_nid_t;
typedef u64 erofs_off_t;
typedef u64 erofs_blk_t;

struct erofs_device_info {
        char *path;
        struct erofs_fscache *fscache;
        struct file *file;
        struct dax_device *dax_dev;
        u64 fsoff, dax_part_off;

        erofs_blk_t blocks;
        erofs_blk_t uniaddr;
};

enum {
        EROFS_SYNC_DECOMPRESS_AUTO,
        EROFS_SYNC_DECOMPRESS_FORCE_ON,
        EROFS_SYNC_DECOMPRESS_FORCE_OFF
};

struct erofs_mount_opts {
        /* current strategy of how to use managed cache */
        unsigned char cache_strategy;
        /* strategy of sync decompression (0 - auto, 1 - force on, 2 - force off) */
        unsigned int sync_decompress;
        /* threshold for decompression synchronously */
        unsigned int max_sync_decompress_pages;
        unsigned int mount_opt;
};

struct erofs_dev_context {
        struct idr tree;
        struct rw_semaphore rwsem;

        unsigned int extra_devices;
        bool flatdev;
};

/* all filesystem-wide lz4 configurations */
struct erofs_sb_lz4_info {
        /* # of pages needed for EROFS lz4 rolling decompression */
        u16 max_distance_pages;
        /* maximum possible blocks for pclusters in the filesystem */
        u16 max_pclusterblks;
};

struct erofs_domain {
        refcount_t ref;
        struct list_head list;
        struct fscache_volume *volume;
        char *domain_id;
};

struct erofs_fscache {
        struct fscache_cookie *cookie;
        struct inode *inode;        /* anonymous inode for the blob */

        /* used for share domain mode */
        struct erofs_domain *domain;
        struct list_head node;
        refcount_t ref;
        char *name;
};

struct erofs_xattr_prefix_item {
        struct erofs_xattr_long_prefix *prefix;
        u8 infix_len;
};

struct erofs_sb_info {
        struct erofs_device_info dif0;
        struct erofs_mount_opts opt;        /* options */
#ifdef CONFIG_EROFS_FS_ZIP
        /* list for all registered superblocks, mainly for shrinker */
        struct list_head list;
        struct mutex umount_mutex;

        /* managed XArray arranged in physical block number */
        struct xarray managed_pslots;

        unsigned int shrinker_run_no;
        u16 available_compr_algs;

        /* pseudo inode to manage cached pages */
        struct inode *managed_cache;

        struct erofs_sb_lz4_info lz4;
#endif        /* CONFIG_EROFS_FS_ZIP */
        struct inode *packed_inode;
        struct erofs_dev_context *devs;
        u64 total_blocks;

        u32 meta_blkaddr;
#ifdef CONFIG_EROFS_FS_XATTR
        u32 xattr_blkaddr;
        u32 xattr_prefix_start;
        u8 xattr_prefix_count;
        struct erofs_xattr_prefix_item *xattr_prefixes;
        unsigned int xattr_filter_reserved;
#endif
        u16 device_id_mask;        /* valid bits of device id to be used */

        unsigned char islotbits;        /* inode slot unit size in bit shift */
        unsigned char blkszbits;        /* filesystem block size in bit shift */

        u32 sb_size;                        /* total superblock size */
        u32 fixed_nsec;
        s64 epoch;

        /* what we really care is nid, rather than ino.. */
        erofs_nid_t root_nid;
        erofs_nid_t packed_nid;
        /* used for statfs, f_files - f_favail */
        u64 inos;

        u32 feature_compat;
        u32 feature_incompat;

        /* sysfs support */
        struct kobject s_kobj;                /* /sys/fs/erofs/<devname> */
        struct completion s_kobj_unregister;

        /* fscache support */
        struct fscache_volume *volume;
        struct erofs_domain *domain;
        char *fsid;
        char *domain_id;
};

#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)

/* Mount flags set via mount options or defaults */
#define EROFS_MOUNT_XATTR_USER                0x00000010
#define EROFS_MOUNT_POSIX_ACL                0x00000020
#define EROFS_MOUNT_DAX_ALWAYS                0x00000040
#define EROFS_MOUNT_DAX_NEVER                0x00000080
#define EROFS_MOUNT_DIRECT_IO                0x00000100

#define clear_opt(opt, option)        ((opt)->mount_opt &= ~EROFS_MOUNT_##option)
#define set_opt(opt, option)        ((opt)->mount_opt |= EROFS_MOUNT_##option)
#define test_opt(opt, option)        ((opt)->mount_opt & EROFS_MOUNT_##option)

static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi)
{
        return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->dif0.file;
}

static inline bool erofs_is_fscache_mode(struct super_block *sb)
{
        return IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) &&
                        !erofs_is_fileio_mode(EROFS_SB(sb)) && !sb->s_bdev;
}

enum {
        EROFS_ZIP_CACHE_DISABLED,
        EROFS_ZIP_CACHE_READAHEAD,
        EROFS_ZIP_CACHE_READAROUND
};

struct erofs_buf {
        struct address_space *mapping;
        struct file *file;
        u64 off;
        struct page *page;
        void *base;
};
#define __EROFS_BUF_INITIALIZER        ((struct erofs_buf){ .page = NULL })

#define erofs_blknr(sb, pos)        ((erofs_blk_t)((pos) >> (sb)->s_blocksize_bits))
#define erofs_blkoff(sb, pos)        ((pos) & ((sb)->s_blocksize - 1))
#define erofs_pos(sb, blk)        ((erofs_off_t)(blk) << (sb)->s_blocksize_bits)
#define erofs_iblks(i)        (round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits)

#define EROFS_FEATURE_FUNCS(name, compat, feature) \
static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \
{ \
        return sbi->feature_##compat & EROFS_FEATURE_##feature; \
}

EROFS_FEATURE_FUNCS(zero_padding, incompat, INCOMPAT_ZERO_PADDING)
EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES)
EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER)

/* atomic flag definitions */
#define EROFS_I_EA_INITED_BIT        0
#define EROFS_I_Z_INITED_BIT        1

/* bitlock definitions (arranged in reverse order) */
#define EROFS_I_BL_XATTR_BIT        (BITS_PER_LONG - 1)
#define EROFS_I_BL_Z_BIT        (BITS_PER_LONG - 2)

struct erofs_inode {
        erofs_nid_t nid;

        /* atomic flags (including bitlocks) */
        unsigned long flags;

        unsigned char datalayout;
        unsigned char inode_isize;
        bool dot_omitted;
        unsigned int xattr_isize;

        unsigned int xattr_name_filter;
        unsigned int xattr_shared_count;
        unsigned int *xattr_shared_xattrs;

        union {
                erofs_blk_t startblk;
                struct {
                        unsigned short        chunkformat;
                        unsigned char        chunkbits;
                };
#ifdef CONFIG_EROFS_FS_ZIP
                struct {
                        unsigned short z_advise;
                        unsigned char  z_algorithmtype[2];
                        unsigned char  z_lclusterbits;
                        union {
                                u64    z_tailextent_headlcn;
                                u64    z_extents;
                        };
                        erofs_off_t    z_fragmentoff;
                        unsigned short z_idata_size;
                };
#endif        /* CONFIG_EROFS_FS_ZIP */
        };
        /* the corresponding vfs inode */
        struct inode vfs_inode;
};

#define EROFS_I(ptr)        container_of(ptr, struct erofs_inode, vfs_inode)

static inline erofs_off_t erofs_iloc(struct inode *inode)
{
        struct erofs_sb_info *sbi = EROFS_I_SB(inode);

        return erofs_pos(inode->i_sb, sbi->meta_blkaddr) +
                (EROFS_I(inode)->nid << sbi->islotbits);
}

static inline unsigned int erofs_inode_version(unsigned int ifmt)
{
        return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK;
}

static inline unsigned int erofs_inode_datalayout(unsigned int ifmt)
{
        return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK;
}

/* reclaiming is never triggered when allocating new folios. */
static inline struct folio *erofs_grab_folio_nowait(struct address_space *as,
                                                    pgoff_t index)
{
        return __filemap_get_folio(as, index,
                        FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
                        readahead_gfp_mask(as) & ~__GFP_RECLAIM);
}

/* Has a disk mapping */
#define EROFS_MAP_MAPPED        0x0001
/* Located in metadata (could be copied from bd_inode) */
#define EROFS_MAP_META                0x0002
/* The extent is encoded */
#define EROFS_MAP_ENCODED        0x0004
/* The length of extent is full */
#define EROFS_MAP_FULL_MAPPED        0x0008
/* Located in the special packed inode */
#define EROFS_MAP_FRAGMENT        0x0010
/* The extent refers to partial decompressed data */
#define EROFS_MAP_PARTIAL_REF        0x0020

struct erofs_map_blocks {
        struct erofs_buf buf;

        erofs_off_t m_pa, m_la;
        u64 m_plen, m_llen;

        unsigned short m_deviceid;
        char m_algorithmformat;
        unsigned int m_flags;
};

/*
 * Used to get the exact decompressed length, e.g. fiemap (consider lookback
 * approach instead if possible since it's more metadata lightweight.)
 */
#define EROFS_GET_BLOCKS_FIEMAP                0x0001
/* Used to map the whole extent if non-negligible data is requested for LZMA */
#define EROFS_GET_BLOCKS_READMORE        0x0002
/* Used to map tail extent for tailpacking inline or fragment pcluster */
#define EROFS_GET_BLOCKS_FINDTAIL        0x0004

enum {
        Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
        Z_EROFS_COMPRESSION_INTERLACED,
        Z_EROFS_COMPRESSION_RUNTIME_MAX
};

struct erofs_map_dev {
        struct super_block *m_sb;
        struct erofs_device_info *m_dif;
        struct block_device *m_bdev;

        erofs_off_t m_pa;
        unsigned int m_deviceid;
};

extern const struct super_operations erofs_sops;

extern const struct address_space_operations erofs_aops;
extern const struct address_space_operations erofs_fileio_aops;
extern const struct address_space_operations z_erofs_aops;
extern const struct address_space_operations erofs_fscache_access_aops;

extern const struct inode_operations erofs_generic_iops;
extern const struct inode_operations erofs_symlink_iops;
extern const struct inode_operations erofs_fast_symlink_iops;
extern const struct inode_operations erofs_dir_iops;

extern const struct file_operations erofs_file_fops;
extern const struct file_operations erofs_dir_fops;

extern const struct iomap_ops z_erofs_iomap_report_ops;

/* flags for erofs_fscache_register_cookie() */
#define EROFS_REG_COOKIE_SHARE                0x0001
#define EROFS_REG_COOKIE_NEED_NOEXIST        0x0002

void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf,
                          erofs_off_t *offset, int *lengthp);
void erofs_unmap_metabuf(struct erofs_buf *buf);
void erofs_put_metabuf(struct erofs_buf *buf);
void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap);
void erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb);
void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb,
                         erofs_off_t offset, bool need_kmap);
int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev);
int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
                 u64 start, u64 len);
int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map);
void erofs_onlinefolio_init(struct folio *folio);
void erofs_onlinefolio_split(struct folio *folio);
void erofs_onlinefolio_end(struct folio *folio, int err);
struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid);
int erofs_getattr(struct mnt_idmap *idmap, const struct path *path,
                  struct kstat *stat, u32 request_mask,
                  unsigned int query_flags);
int erofs_namei(struct inode *dir, const struct qstr *name,
                erofs_nid_t *nid, unsigned int *d_type);

static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count)
{
        int retried = 0;

        while (1) {
                void *p = vm_map_ram(pages, count, -1);

                /* retry two more times (totally 3 times) */
                if (p || ++retried >= 3)
                        return p;
                vm_unmap_aliases();
        }
        return NULL;
}

int erofs_register_sysfs(struct super_block *sb);
void erofs_unregister_sysfs(struct super_block *sb);
int __init erofs_init_sysfs(void);
void erofs_exit_sysfs(void);

struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv);
static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp)
{
        return __erofs_allocpage(pagepool, gfp, false);
}
static inline void erofs_pagepool_add(struct page **pagepool, struct page *page)
{
        set_page_private(page, (unsigned long)*pagepool);
        *pagepool = page;
}
void erofs_release_pages(struct page **pagepool);

#ifdef CONFIG_EROFS_FS_ZIP
#define MNGD_MAPPING(sbi)        ((sbi)->managed_cache->i_mapping)

extern atomic_long_t erofs_global_shrink_cnt;
void erofs_shrinker_register(struct super_block *sb);
void erofs_shrinker_unregister(struct super_block *sb);
int __init erofs_init_shrinker(void);
void erofs_exit_shrinker(void);
int __init z_erofs_init_subsystem(void);
void z_erofs_exit_subsystem(void);
int z_erofs_init_super(struct super_block *sb);
unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi,
                                  unsigned long nr_shrink);
int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
                            int flags);
void *z_erofs_get_gbuf(unsigned int requiredpages);
void z_erofs_put_gbuf(void *ptr);
int z_erofs_gbuf_growsize(unsigned int nrpages);
int __init z_erofs_gbuf_init(void);
void z_erofs_gbuf_exit(void);
int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb);
#else
static inline void erofs_shrinker_register(struct super_block *sb) {}
static inline void erofs_shrinker_unregister(struct super_block *sb) {}
static inline int erofs_init_shrinker(void) { return 0; }
static inline void erofs_exit_shrinker(void) {}
static inline int z_erofs_init_subsystem(void) { return 0; }
static inline void z_erofs_exit_subsystem(void) {}
static inline int z_erofs_init_super(struct super_block *sb) { return 0; }
#endif        /* !CONFIG_EROFS_FS_ZIP */

#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE
struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev);
void erofs_fileio_submit_bio(struct bio *bio);
#else
static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { return NULL; }
static inline void erofs_fileio_submit_bio(struct bio *bio) {}
#endif

#ifdef CONFIG_EROFS_FS_ONDEMAND
int erofs_fscache_register_fs(struct super_block *sb);
void erofs_fscache_unregister_fs(struct super_block *sb);

struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
                                        char *name, unsigned int flags);
void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache);
struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev);
void erofs_fscache_submit_bio(struct bio *bio);
#else
static inline int erofs_fscache_register_fs(struct super_block *sb)
{
        return -EOPNOTSUPP;
}
static inline void erofs_fscache_unregister_fs(struct super_block *sb) {}

static inline
struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
                                        char *name, unsigned int flags)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache)
{
}
static inline struct bio *erofs_fscache_bio_alloc(struct erofs_map_dev *mdev) { return NULL; }
static inline void erofs_fscache_submit_bio(struct bio *bio) {}
#endif

#define EFSCORRUPTED    EUCLEAN         /* Filesystem is corrupted */

#endif        /* __EROFS_INTERNAL_H */













    2 






  310 





    2 





    2 






    2 










   40 





   40 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#include <linux/notifier.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <net/net_namespace.h>
#include <net/fib_notifier.h>
#include <net/netns/ipv6.h>
#include <net/ip6_fib.h>

int call_fib6_notifier(struct notifier_block *nb,
                       enum fib_event_type event_type,
                       struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifier(nb, event_type, info);
}

int call_fib6_notifiers(struct net *net, enum fib_event_type event_type,
                        struct fib_notifier_info *info)
{
        info->family = AF_INET6;
        return call_fib_notifiers(net, event_type, info);
}

static unsigned int fib6_seq_read(const struct net *net)
{
        return fib6_tables_seq_read(net) + fib6_rules_seq_read(net);
}

static int fib6_dump(struct net *net, struct notifier_block *nb,
                     struct netlink_ext_ack *extack)
{
        int err;

        err = fib6_rules_dump(net, nb, extack);
        if (err)
                return err;

        return fib6_tables_dump(net, nb, extack);
}

static const struct fib_notifier_ops fib6_notifier_ops_template = {
        .family                = AF_INET6,
        .fib_seq_read        = fib6_seq_read,
        .fib_dump        = fib6_dump,
        .owner                = THIS_MODULE,
};

int __net_init fib6_notifier_init(struct net *net)
{
        struct fib_notifier_ops *ops;

        ops = fib_notifier_ops_register(&fib6_notifier_ops_template, net);
        if (IS_ERR(ops))
                return PTR_ERR(ops);
        net->ipv6.notifier_ops = ops;

        return 0;
}

void __net_exit fib6_notifier_exit(struct net *net)
{
        fib_notifier_ops_unregister(net->ipv6.notifier_ops);
}



































    7 




























































































































































































































































































































































































































































































































































































































































































































































    7 
































    7 









    7 























































































































































































































































































































































































































































































































































































   17 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
// SPDX-License-Identifier: GPL-2.0

#include <linux/err.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "messages.h"
#include "ctree.h"
#include "extent_map.h"
#include "compression.h"
#include "btrfs_inode.h"
#include "disk-io.h"


static struct kmem_cache *extent_map_cache;

int __init btrfs_extent_map_init(void)
{
        extent_map_cache = kmem_cache_create("btrfs_extent_map",
                                             sizeof(struct extent_map), 0, 0, NULL);
        if (!extent_map_cache)
                return -ENOMEM;
        return 0;
}

void __cold btrfs_extent_map_exit(void)
{
        kmem_cache_destroy(extent_map_cache);
}

/*
 * Initialize the extent tree @tree.  Should be called for each new inode or
 * other user of the extent_map interface.
 */
void btrfs_extent_map_tree_init(struct extent_map_tree *tree)
{
        tree->root = RB_ROOT;
        INIT_LIST_HEAD(&tree->modified_extents);
        rwlock_init(&tree->lock);
}

/*
 * Allocate a new extent_map structure.  The new structure is returned with a
 * reference count of one and needs to be freed using free_extent_map()
 */
struct extent_map *btrfs_alloc_extent_map(void)
{
        struct extent_map *em;
        em = kmem_cache_zalloc(extent_map_cache, GFP_NOFS);
        if (!em)
                return NULL;
        RB_CLEAR_NODE(&em->rb_node);
        refcount_set(&em->refs, 1);
        INIT_LIST_HEAD(&em->list);
        return em;
}

/*
 * Drop the reference out on @em by one and free the structure if the reference
 * count hits zero.
 */
void btrfs_free_extent_map(struct extent_map *em)
{
        if (!em)
                return;
        if (refcount_dec_and_test(&em->refs)) {
                WARN_ON(btrfs_extent_map_in_tree(em));
                WARN_ON(!list_empty(&em->list));
                kmem_cache_free(extent_map_cache, em);
        }
}

/* Do the math around the end of an extent, handling wrapping. */
static u64 range_end(u64 start, u64 len)
{
        if (start + len < start)
                return (u64)-1;
        return start + len;
}

static void remove_em(struct btrfs_inode *inode, struct extent_map *em)
{
        struct btrfs_fs_info *fs_info = inode->root->fs_info;

        rb_erase(&em->rb_node, &inode->extent_tree.root);
        RB_CLEAR_NODE(&em->rb_node);

        if (!btrfs_is_testing(fs_info) && is_fstree(btrfs_root_id(inode->root)))
                percpu_counter_dec(&fs_info->evictable_extent_maps);
}

static int tree_insert(struct rb_root *root, struct extent_map *em)
{
        struct rb_node **p = &root->rb_node;
        struct rb_node *parent = NULL;
        struct extent_map *entry = NULL;
        struct rb_node *orig_parent = NULL;
        u64 end = range_end(em->start, em->len);

        while (*p) {
                parent = *p;
                entry = rb_entry(parent, struct extent_map, rb_node);

                if (em->start < entry->start)
                        p = &(*p)->rb_left;
                else if (em->start >= btrfs_extent_map_end(entry))
                        p = &(*p)->rb_right;
                else
                        return -EEXIST;
        }

        orig_parent = parent;
        while (parent && em->start >= btrfs_extent_map_end(entry)) {
                parent = rb_next(parent);
                entry = rb_entry(parent, struct extent_map, rb_node);
        }
        if (parent)
                if (end > entry->start && em->start < btrfs_extent_map_end(entry))
                        return -EEXIST;

        parent = orig_parent;
        entry = rb_entry(parent, struct extent_map, rb_node);
        while (parent && em->start < entry->start) {
                parent = rb_prev(parent);
                entry = rb_entry(parent, struct extent_map, rb_node);
        }
        if (parent)
                if (end > entry->start && em->start < btrfs_extent_map_end(entry))
                        return -EEXIST;

        rb_link_node(&em->rb_node, orig_parent, p);
        rb_insert_color(&em->rb_node, root);
        return 0;
}

/*
 * Search through the tree for an extent_map with a given offset.  If it can't
 * be found, try to find some neighboring extents
 */
static struct rb_node *tree_search(struct rb_root *root, u64 offset,
                                   struct rb_node **prev_or_next_ret)
{
        struct rb_node *n = root->rb_node;
        struct rb_node *prev = NULL;
        struct rb_node *orig_prev = NULL;
        struct extent_map *entry;
        struct extent_map *prev_entry = NULL;

        ASSERT(prev_or_next_ret);

        while (n) {
                entry = rb_entry(n, struct extent_map, rb_node);
                prev = n;
                prev_entry = entry;

                if (offset < entry->start)
                        n = n->rb_left;
                else if (offset >= btrfs_extent_map_end(entry))
                        n = n->rb_right;
                else
                        return n;
        }

        orig_prev = prev;
        while (prev && offset >= btrfs_extent_map_end(prev_entry)) {
                prev = rb_next(prev);
                prev_entry = rb_entry(prev, struct extent_map, rb_node);
        }

        /*
         * Previous extent map found, return as in this case the caller does not
         * care about the next one.
         */
        if (prev) {
                *prev_or_next_ret = prev;
                return NULL;
        }

        prev = orig_prev;
        prev_entry = rb_entry(prev, struct extent_map, rb_node);
        while (prev && offset < prev_entry->start) {
                prev = rb_prev(prev);
                prev_entry = rb_entry(prev, struct extent_map, rb_node);
        }
        *prev_or_next_ret = prev;

        return NULL;
}

static inline u64 extent_map_block_len(const struct extent_map *em)
{
        if (btrfs_extent_map_is_compressed(em))
                return em->disk_num_bytes;
        return em->len;
}

static inline u64 extent_map_block_end(const struct extent_map *em)
{
        const u64 block_start = btrfs_extent_map_block_start(em);
        const u64 block_end = block_start + extent_map_block_len(em);

        if (block_end < block_start)
                return (u64)-1;

        return block_end;
}

static bool can_merge_extent_map(const struct extent_map *em)
{
        if (em->flags & EXTENT_FLAG_PINNED)
                return false;

        /* Don't merge compressed extents, we need to know their actual size. */
        if (btrfs_extent_map_is_compressed(em))
                return false;

        if (em->flags & EXTENT_FLAG_LOGGING)
                return false;

        /*
         * We don't want to merge stuff that hasn't been written to the log yet
         * since it may not reflect exactly what is on disk, and that would be
         * bad.
         */
        if (!list_empty(&em->list))
                return false;

        return true;
}

/* Check to see if two extent_map structs are adjacent and safe to merge. */
static bool mergeable_maps(const struct extent_map *prev, const struct extent_map *next)
{
        if (btrfs_extent_map_end(prev) != next->start)
                return false;

        /*
         * The merged flag is not an on-disk flag, it just indicates we had the
         * extent maps of 2 (or more) adjacent extents merged, so factor it out.
         */
        if ((prev->flags & ~EXTENT_FLAG_MERGED) !=
            (next->flags & ~EXTENT_FLAG_MERGED))
                return false;

        if (next->disk_bytenr < EXTENT_MAP_LAST_BYTE - 1)
                return btrfs_extent_map_block_start(next) == extent_map_block_end(prev);

        /* HOLES and INLINE extents. */
        return next->disk_bytenr == prev->disk_bytenr;
}

/*
 * Handle the on-disk data extents merge for @prev and @next.
 *
 * @prev:    left extent to merge
 * @next:    right extent to merge
 * @merged:  the extent we will not discard after the merge; updated with new values
 *
 * After this, one of the two extents is the new merged extent and the other is
 * removed from the tree and likely freed. Note that @merged is one of @prev/@next
 * so there is const/non-const aliasing occurring here.
 *
 * Only touches disk_bytenr/disk_num_bytes/offset/ram_bytes.
 * For now only uncompressed regular extent can be merged.
 */
static void merge_ondisk_extents(const struct extent_map *prev, const struct extent_map *next,
                                 struct extent_map *merged)
{
        u64 new_disk_bytenr;
        u64 new_disk_num_bytes;
        u64 new_offset;

        /* @prev and @next should not be compressed. */
        ASSERT(!btrfs_extent_map_is_compressed(prev));
        ASSERT(!btrfs_extent_map_is_compressed(next));

        /*
         * There are two different cases where @prev and @next can be merged.
         *
         * 1) They are referring to the same data extent:
         *
         * |<----- data extent A ----->|
         *    |<- prev ->|<- next ->|
         *
         * 2) They are referring to different data extents but still adjacent:
         *
         * |<-- data extent A -->|<-- data extent B -->|
         *            |<- prev ->|<- next ->|
         *
         * The calculation here always merges the data extents first, then updates
         * @offset using the new data extents.
         *
         * For case 1), the merged data extent would be the same.
         * For case 2), we just merge the two data extents into one.
         */
        new_disk_bytenr = min(prev->disk_bytenr, next->disk_bytenr);
        new_disk_num_bytes = max(prev->disk_bytenr + prev->disk_num_bytes,
                                 next->disk_bytenr + next->disk_num_bytes) -
                             new_disk_bytenr;
        new_offset = prev->disk_bytenr + prev->offset - new_disk_bytenr;

        merged->disk_bytenr = new_disk_bytenr;
        merged->disk_num_bytes = new_disk_num_bytes;
        merged->ram_bytes = new_disk_num_bytes;
        merged->offset = new_offset;
}

static void dump_extent_map(struct btrfs_fs_info *fs_info, const char *prefix,
                            struct extent_map *em)
{
        if (!IS_ENABLED(CONFIG_BTRFS_DEBUG))
                return;
        btrfs_crit(fs_info,
"%s, start=%llu len=%llu disk_bytenr=%llu disk_num_bytes=%llu ram_bytes=%llu offset=%llu flags=0x%x",
                prefix, em->start, em->len, em->disk_bytenr, em->disk_num_bytes,
                em->ram_bytes, em->offset, em->flags);
        ASSERT(0);
}

/* Internal sanity checks for btrfs debug builds. */
static void validate_extent_map(struct btrfs_fs_info *fs_info, struct extent_map *em)
{
        if (!IS_ENABLED(CONFIG_BTRFS_DEBUG))
                return;
        if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {
                if (em->disk_num_bytes == 0)
                        dump_extent_map(fs_info, "zero disk_num_bytes", em);
                if (em->offset + em->len > em->ram_bytes)
                        dump_extent_map(fs_info, "ram_bytes too small", em);
                if (em->offset + em->len > em->disk_num_bytes &&
                    !btrfs_extent_map_is_compressed(em))
                        dump_extent_map(fs_info, "disk_num_bytes too small", em);
                if (!btrfs_extent_map_is_compressed(em) &&
                    em->ram_bytes != em->disk_num_bytes)
                        dump_extent_map(fs_info,
                "ram_bytes mismatch with disk_num_bytes for non-compressed em",
                                        em);
        } else if (em->offset) {
                dump_extent_map(fs_info, "non-zero offset for hole/inline", em);
        }
}

static void try_merge_map(struct btrfs_inode *inode, struct extent_map *em)
{
        struct btrfs_fs_info *fs_info = inode->root->fs_info;
        struct extent_map *merge = NULL;
        struct rb_node *rb;

        /*
         * We can't modify an extent map that is in the tree and that is being
         * used by another task, as it can cause that other task to see it in
         * inconsistent state during the merging. We always have 1 reference for
         * the tree and 1 for this task (which is unpinning the extent map or
         * clearing the logging flag), so anything > 2 means it's being used by
         * other tasks too.
         */
        if (refcount_read(&em->refs) > 2)
                return;

        if (!can_merge_extent_map(em))
                return;

        if (em->start != 0) {
                rb = rb_prev(&em->rb_node);
                merge = rb_entry_safe(rb, struct extent_map, rb_node);

                if (rb && can_merge_extent_map(merge) && mergeable_maps(merge, em)) {
                        em->start = merge->start;
                        em->len += merge->len;
                        em->generation = max(em->generation, merge->generation);

                        if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
                                merge_ondisk_extents(merge, em, em);
                        em->flags |= EXTENT_FLAG_MERGED;

                        validate_extent_map(fs_info, em);
                        remove_em(inode, merge);
                        btrfs_free_extent_map(merge);
                }
        }

        rb = rb_next(&em->rb_node);
        merge = rb_entry_safe(rb, struct extent_map, rb_node);

        if (rb && can_merge_extent_map(merge) && mergeable_maps(em, merge)) {
                em->len += merge->len;
                if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
                        merge_ondisk_extents(em, merge, em);
                validate_extent_map(fs_info, em);
                em->generation = max(em->generation, merge->generation);
                em->flags |= EXTENT_FLAG_MERGED;
                remove_em(inode, merge);
                btrfs_free_extent_map(merge);
        }
}

/*
 * Unpin an extent from the cache.
 *
 * @inode:        the inode from which we are unpinning an extent range
 * @start:        logical offset in the file
 * @len:        length of the extent
 * @gen:        generation that this extent has been modified in
 *
 * Called after an extent has been written to disk properly.  Set the generation
 * to the generation that actually added the file item to the inode so we know
 * we need to sync this extent when we call fsync().
 *
 * Returns: 0             on success
 *             -ENOENT  when the extent is not found in the tree
 *             -EUCLEAN if the found extent does not match the expected start
 */
int btrfs_unpin_extent_cache(struct btrfs_inode *inode, u64 start, u64 len, u64 gen)
{
        struct btrfs_fs_info *fs_info = inode->root->fs_info;
        struct extent_map_tree *tree = &inode->extent_tree;
        int ret = 0;
        struct extent_map *em;

        write_lock(&tree->lock);
        em = btrfs_lookup_extent_mapping(tree, start, len);

        if (WARN_ON(!em)) {
                btrfs_warn(fs_info,
"no extent map found for inode %llu (root %lld) when unpinning extent range [%llu, %llu), generation %llu",
                           btrfs_ino(inode), btrfs_root_id(inode->root),
                           start, start + len, gen);
                ret = -ENOENT;
                goto out;
        }

        if (WARN_ON(em->start != start)) {
                btrfs_warn(fs_info,
"found extent map for inode %llu (root %lld) with unexpected start offset %llu when unpinning extent range [%llu, %llu), generation %llu",
                           btrfs_ino(inode), btrfs_root_id(inode->root),
                           em->start, start, start + len, gen);
                ret = -EUCLEAN;
                goto out;
        }

        em->generation = gen;
        em->flags &= ~EXTENT_FLAG_PINNED;

        try_merge_map(inode, em);

out:
        write_unlock(&tree->lock);
        btrfs_free_extent_map(em);
        return ret;

}

void btrfs_clear_em_logging(struct btrfs_inode *inode, struct extent_map *em)
{
        lockdep_assert_held_write(&inode->extent_tree.lock);

        em->flags &= ~EXTENT_FLAG_LOGGING;
        if (btrfs_extent_map_in_tree(em))
                try_merge_map(inode, em);
}

static inline void setup_extent_mapping(struct btrfs_inode *inode,
                                        struct extent_map *em,
                                        int modified)
{
        refcount_inc(&em->refs);

        ASSERT(list_empty(&em->list));

        if (modified)
                list_add(&em->list, &inode->extent_tree.modified_extents);
        else
                try_merge_map(inode, em);
}

/*
 * Add a new extent map to an inode's extent map tree.
 *
 * @inode:        the target inode
 * @em:                map to insert
 * @modified:        indicate whether the given @em should be added to the
 *                modified list, which indicates the extent needs to be logged
 *
 * Insert @em into the @inode's extent map tree or perform a simple
 * forward/backward merge with existing mappings.  The extent_map struct passed
 * in will be inserted into the tree directly, with an additional reference
 * taken, or a reference dropped if the merge attempt was successful.
 */
static int add_extent_mapping(struct btrfs_inode *inode,
                              struct extent_map *em, int modified)
{
        struct extent_map_tree *tree = &inode->extent_tree;
        struct btrfs_root *root = inode->root;
        struct btrfs_fs_info *fs_info = root->fs_info;
        int ret;

        lockdep_assert_held_write(&tree->lock);

        validate_extent_map(fs_info, em);
        ret = tree_insert(&tree->root, em);
        if (ret)
                return ret;

        setup_extent_mapping(inode, em, modified);

        if (!btrfs_is_testing(fs_info) && is_fstree(btrfs_root_id(root)))
                percpu_counter_inc(&fs_info->evictable_extent_maps);

        return 0;
}

static struct extent_map *lookup_extent_mapping(struct extent_map_tree *tree,
                                                u64 start, u64 len, int strict)
{
        struct extent_map *em;
        struct rb_node *rb_node;
        struct rb_node *prev_or_next = NULL;
        u64 end = range_end(start, len);

        rb_node = tree_search(&tree->root, start, &prev_or_next);
        if (!rb_node) {
                if (prev_or_next)
                        rb_node = prev_or_next;
                else
                        return NULL;
        }

        em = rb_entry(rb_node, struct extent_map, rb_node);

        if (strict && !(end > em->start && start < btrfs_extent_map_end(em)))
                return NULL;

        refcount_inc(&em->refs);
        return em;
}

/*
 * Lookup extent_map that intersects @start + @len range.
 *
 * @tree:        tree to lookup in
 * @start:        byte offset to start the search
 * @len:        length of the lookup range
 *
 * Find and return the first extent_map struct in @tree that intersects the
 * [start, len] range.  There may be additional objects in the tree that
 * intersect, so check the object returned carefully to make sure that no
 * additional lookups are needed.
 */
struct extent_map *btrfs_lookup_extent_mapping(struct extent_map_tree *tree,
                                               u64 start, u64 len)
{
        return lookup_extent_mapping(tree, start, len, 1);
}

/*
 * Find a nearby extent map intersecting @start + @len (not an exact search).
 *
 * @tree:        tree to lookup in
 * @start:        byte offset to start the search
 * @len:        length of the lookup range
 *
 * Find and return the first extent_map struct in @tree that intersects the
 * [start, len] range.
 *
 * If one can't be found, any nearby extent may be returned
 */
struct extent_map *btrfs_search_extent_mapping(struct extent_map_tree *tree,
                                               u64 start, u64 len)
{
        return lookup_extent_mapping(tree, start, len, 0);
}

/*
 * Remove an extent_map from its inode's extent tree.
 *
 * @inode:        the inode the extent map belongs to
 * @em:                extent map being removed
 *
 * Remove @em from the extent tree of @inode.  No reference counts are dropped,
 * and no checks are done to see if the range is in use.
 */
void btrfs_remove_extent_mapping(struct btrfs_inode *inode, struct extent_map *em)
{
        struct extent_map_tree *tree = &inode->extent_tree;

        lockdep_assert_held_write(&tree->lock);

        WARN_ON(em->flags & EXTENT_FLAG_PINNED);
        if (!(em->flags & EXTENT_FLAG_LOGGING))
                list_del_init(&em->list);

        remove_em(inode, em);
}

static void replace_extent_mapping(struct btrfs_inode *inode,
                                   struct extent_map *cur,
                                   struct extent_map *new,
                                   int modified)
{
        struct btrfs_fs_info *fs_info = inode->root->fs_info;
        struct extent_map_tree *tree = &inode->extent_tree;

        lockdep_assert_held_write(&tree->lock);

        validate_extent_map(fs_info, new);

        WARN_ON(cur->flags & EXTENT_FLAG_PINNED);
        ASSERT(btrfs_extent_map_in_tree(cur));
        if (!(cur->flags & EXTENT_FLAG_LOGGING))
                list_del_init(&cur->list);
        rb_replace_node(&cur->rb_node, &new->rb_node, &tree->root);
        RB_CLEAR_NODE(&cur->rb_node);

        setup_extent_mapping(inode, new, modified);
}

static struct extent_map *next_extent_map(const struct extent_map *em)
{
        struct rb_node *next;

        next = rb_next(&em->rb_node);
        if (!next)
                return NULL;
        return container_of(next, struct extent_map, rb_node);
}

static struct extent_map *prev_extent_map(struct extent_map *em)
{
        struct rb_node *prev;

        prev = rb_prev(&em->rb_node);
        if (!prev)
                return NULL;
        return container_of(prev, struct extent_map, rb_node);
}

/*
 * Helper for btrfs_get_extent.  Given an existing extent in the tree,
 * the existing extent is the nearest extent to map_start,
 * and an extent that you want to insert, deal with overlap and insert
 * the best fitted new extent into the tree.
 */
static noinline int merge_extent_mapping(struct btrfs_inode *inode,
                                         struct extent_map *existing,
                                         struct extent_map *em,
                                         u64 map_start)
{
        struct extent_map *prev;
        struct extent_map *next;
        u64 start;
        u64 end;
        u64 start_diff;

        if (map_start < em->start || map_start >= btrfs_extent_map_end(em))
                return -EINVAL;

        if (existing->start > map_start) {
                next = existing;
                prev = prev_extent_map(next);
        } else {
                prev = existing;
                next = next_extent_map(prev);
        }

        start = prev ? btrfs_extent_map_end(prev) : em->start;
        start = max_t(u64, start, em->start);
        end = next ? next->start : btrfs_extent_map_end(em);
        end = min_t(u64, end, btrfs_extent_map_end(em));
        start_diff = start - em->start;
        em->start = start;
        em->len = end - start;
        if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE)
                em->offset += start_diff;
        return add_extent_mapping(inode, em, 0);
}

/*
 * Add extent mapping into an inode's extent map tree.
 *
 * @inode:    target inode
 * @em_in:    extent we are inserting
 * @start:    start of the logical range btrfs_get_extent() is requesting
 * @len:      length of the logical range btrfs_get_extent() is requesting
 *
 * Note that @em_in's range may be different from [start, start+len),
 * but they must be overlapped.
 *
 * Insert @em_in into the inode's extent map tree. In case there is an
 * overlapping range, handle the -EEXIST by either:
 * a) Returning the existing extent in @em_in if @start is within the
 *    existing em.
 * b) Merge the existing extent with @em_in passed in.
 *
 * Return 0 on success, otherwise -EEXIST.
 *
 */
int btrfs_add_extent_mapping(struct btrfs_inode *inode,
                             struct extent_map **em_in, u64 start, u64 len)
{
        int ret;
        struct extent_map *em = *em_in;
        struct btrfs_fs_info *fs_info = inode->root->fs_info;

        /*
         * Tree-checker should have rejected any inline extent with non-zero
         * file offset. Here just do a sanity check.
         */
        if (em->disk_bytenr == EXTENT_MAP_INLINE)
                ASSERT(em->start == 0);

        ret = add_extent_mapping(inode, em, 0);
        /* it is possible that someone inserted the extent into the tree
         * while we had the lock dropped.  It is also possible that
         * an overlapping map exists in the tree
         */
        if (ret == -EEXIST) {
                struct extent_map *existing;

                existing = btrfs_search_extent_mapping(&inode->extent_tree, start, len);

                trace_btrfs_handle_em_exist(fs_info, existing, em, start, len);

                /*
                 * existing will always be non-NULL, since there must be
                 * extent causing the -EEXIST.
                 */
                if (start >= existing->start &&
                    start < btrfs_extent_map_end(existing)) {
                        btrfs_free_extent_map(em);
                        *em_in = existing;
                        ret = 0;
                } else {
                        u64 orig_start = em->start;
                        u64 orig_len = em->len;

                        /*
                         * The existing extent map is the one nearest to
                         * the [start, start + len) range which overlaps
                         */
                        ret = merge_extent_mapping(inode, existing, em, start);
                        if (WARN_ON(ret)) {
                                btrfs_free_extent_map(em);
                                *em_in = NULL;
                                btrfs_warn(fs_info,
"extent map merge error existing [%llu, %llu) with em [%llu, %llu) start %llu",
                                           existing->start, btrfs_extent_map_end(existing),
                                           orig_start, orig_start + orig_len, start);
                        }
                        btrfs_free_extent_map(existing);
                }
        }

        ASSERT(ret == 0 || ret == -EEXIST);
        return ret;
}

/*
 * Drop all extent maps from a tree in the fastest possible way, rescheduling
 * if needed. This avoids searching the tree, from the root down to the first
 * extent map, before each deletion.
 */
static void drop_all_extent_maps_fast(struct btrfs_inode *inode)
{
        struct extent_map_tree *tree = &inode->extent_tree;
        struct rb_node *node;

        write_lock(&tree->lock);
        node = rb_first(&tree->root);
        while (node) {
                struct extent_map *em;
                struct rb_node *next = rb_next(node);

                em = rb_entry(node, struct extent_map, rb_node);
                em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING);
                btrfs_remove_extent_mapping(inode, em);
                btrfs_free_extent_map(em);

                if (cond_resched_rwlock_write(&tree->lock))
                        node = rb_first(&tree->root);
                else
                        node = next;
        }
        write_unlock(&tree->lock);
}

/*
 * Drop all extent maps in a given range.
 *
 * @inode:       The target inode.
 * @start:       Start offset of the range.
 * @end:         End offset of the range (inclusive value).
 * @skip_pinned: Indicate if pinned extent maps should be ignored or not.
 *
 * This drops all the extent maps that intersect the given range [@start, @end].
 * Extent maps that partially overlap the range and extend behind or beyond it,
 * are split.
 * The caller should have locked an appropriate file range in the inode's io
 * tree before calling this function.
 */
void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end,
                                 bool skip_pinned)
{
        struct extent_map *split;
        struct extent_map *split2;
        struct extent_map *em;
        struct extent_map_tree *em_tree = &inode->extent_tree;
        u64 len = end - start + 1;

        WARN_ON(end < start);
        if (end == (u64)-1) {
                if (start == 0 && !skip_pinned) {
                        drop_all_extent_maps_fast(inode);
                        return;
                }
                len = (u64)-1;
        } else {
                /* Make end offset exclusive for use in the loop below. */
                end++;
        }

        /*
         * It's ok if we fail to allocate the extent maps, see the comment near
         * the bottom of the loop below. We only need two spare extent maps in
         * the worst case, where the first extent map that intersects our range
         * starts before the range and the last extent map that intersects our
         * range ends after our range (and they might be the same extent map),
         * because we need to split those two extent maps at the boundaries.
         */
        split = btrfs_alloc_extent_map();
        split2 = btrfs_alloc_extent_map();

        write_lock(&em_tree->lock);
        em = btrfs_lookup_extent_mapping(em_tree, start, len);

        while (em) {
                /* extent_map_end() returns exclusive value (last byte + 1). */
                const u64 em_end = btrfs_extent_map_end(em);
                struct extent_map *next_em = NULL;
                u64 gen;
                unsigned long flags;
                bool modified;

                if (em_end < end) {
                        next_em = next_extent_map(em);
                        if (next_em) {
                                if (next_em->start < end)
                                        refcount_inc(&next_em->refs);
                                else
                                        next_em = NULL;
                        }
                }

                if (skip_pinned && (em->flags & EXTENT_FLAG_PINNED)) {
                        start = em_end;
                        goto next;
                }

                flags = em->flags;
                /*
                 * In case we split the extent map, we want to preserve the
                 * EXTENT_FLAG_LOGGING flag on our extent map, but we don't want
                 * it on the new extent maps.
                 */
                em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING);
                modified = !list_empty(&em->list);

                /*
                 * The extent map does not cross our target range, so no need to
                 * split it, we can remove it directly.
                 */
                if (em->start >= start && em_end <= end)
                        goto remove_em;

                gen = em->generation;

                if (em->start < start) {
                        if (!split) {
                                split = split2;
                                split2 = NULL;
                                if (!split)
                                        goto remove_em;
                        }
                        split->start = em->start;
                        split->len = start - em->start;

                        if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {
                                split->disk_bytenr = em->disk_bytenr;
                                split->disk_num_bytes = em->disk_num_bytes;
                                split->offset = em->offset;
                                split->ram_bytes = em->ram_bytes;
                        } else {
                                split->disk_bytenr = em->disk_bytenr;
                                split->disk_num_bytes = 0;
                                split->offset = 0;
                                split->ram_bytes = split->len;
                        }

                        split->generation = gen;
                        split->flags = flags;
                        replace_extent_mapping(inode, em, split, modified);
                        btrfs_free_extent_map(split);
                        split = split2;
                        split2 = NULL;
                }
                if (em_end > end) {
                        if (!split) {
                                split = split2;
                                split2 = NULL;
                                if (!split)
                                        goto remove_em;
                        }
                        split->start = end;
                        split->len = em_end - end;
                        split->disk_bytenr = em->disk_bytenr;
                        split->flags = flags;
                        split->generation = gen;

                        if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) {
                                split->disk_num_bytes = em->disk_num_bytes;
                                split->offset = em->offset + end - em->start;
                                split->ram_bytes = em->ram_bytes;
                        } else {
                                split->disk_num_bytes = 0;
                                split->offset = 0;
                                split->ram_bytes = split->len;
                        }

                        if (btrfs_extent_map_in_tree(em)) {
                                replace_extent_mapping(inode, em, split, modified);
                        } else {
                                int ret;

                                ret = add_extent_mapping(inode, split, modified);
                                /* Logic error, shouldn't happen. */
                                ASSERT(ret == 0);
                                if (WARN_ON(ret != 0) && modified)
                                        btrfs_set_inode_full_sync(inode);
                        }
                        btrfs_free_extent_map(split);
                        split = NULL;
                }
remove_em:
                if (btrfs_extent_map_in_tree(em)) {
                        /*
                         * If the extent map is still in the tree it means that
                         * either of the following is true:
                         *
                         * 1) It fits entirely in our range (doesn't end beyond
                         *    it or starts before it);
                         *
                         * 2) It starts before our range and/or ends after our
                         *    range, and we were not able to allocate the extent
                         *    maps for split operations, @split and @split2.
                         *
                         * If we are at case 2) then we just remove the entire
                         * extent map - this is fine since if anyone needs it to
                         * access the subranges outside our range, will just
                         * load it again from the subvolume tree's file extent
                         * item. However if the extent map was in the list of
                         * modified extents, then we must mark the inode for a
                         * full fsync, otherwise a fast fsync will miss this
                         * extent if it's new and needs to be logged.
                         */
                        if ((em->start < start || em_end > end) && modified) {
                                ASSERT(!split);
                                btrfs_set_inode_full_sync(inode);
                        }
                        btrfs_remove_extent_mapping(inode, em);
                }

                /*
                 * Once for the tree reference (we replaced or removed the
                 * extent map from the tree).
                 */
                btrfs_free_extent_map(em);
next:
                /* Once for us (for our lookup reference). */
                btrfs_free_extent_map(em);

                em = next_em;
        }

        write_unlock(&em_tree->lock);

        btrfs_free_extent_map(split);
        btrfs_free_extent_map(split2);
}

/*
 * Replace a range in the inode's extent map tree with a new extent map.
 *
 * @inode:      The target inode.
 * @new_em:     The new extent map to add to the inode's extent map tree.
 * @modified:   Indicate if the new extent map should be added to the list of
 *              modified extents (for fast fsync tracking).
 *
 * Drops all the extent maps in the inode's extent map tree that intersect the
 * range of the new extent map and adds the new extent map to the tree.
 * The caller should have locked an appropriate file range in the inode's io
 * tree before calling this function.
 */
int btrfs_replace_extent_map_range(struct btrfs_inode *inode,
                                   struct extent_map *new_em,
                                   bool modified)
{
        const u64 end = new_em->start + new_em->len - 1;
        struct extent_map_tree *tree = &inode->extent_tree;
        int ret;

        ASSERT(!btrfs_extent_map_in_tree(new_em));

        /*
         * The caller has locked an appropriate file range in the inode's io
         * tree, but getting -EEXIST when adding the new extent map can still
         * happen in case there are extents that partially cover the range, and
         * this is due to two tasks operating on different parts of the extent.
         * See commit 18e83ac75bfe67 ("Btrfs: fix unexpected EEXIST from
         * btrfs_get_extent") for an example and details.
         */
        do {
                btrfs_drop_extent_map_range(inode, new_em->start, end, false);
                write_lock(&tree->lock);
                ret = add_extent_mapping(inode, new_em, modified);
                write_unlock(&tree->lock);
        } while (ret == -EEXIST);

        return ret;
}

/*
 * Split off the first pre bytes from the extent_map at [start, start + len],
 * and set the block_start for it to new_logical.
 *
 * This function is used when an ordered_extent needs to be split.
 */
int btrfs_split_extent_map(struct btrfs_inode *inode, u64 start, u64 len, u64 pre,
                           u64 new_logical)
{
        struct extent_map_tree *em_tree = &inode->extent_tree;
        struct extent_map *em;
        struct extent_map *split_pre = NULL;
        struct extent_map *split_mid = NULL;
        int ret = 0;
        unsigned long flags;

        ASSERT(pre != 0);
        ASSERT(pre < len);

        split_pre = btrfs_alloc_extent_map();
        if (!split_pre)
                return -ENOMEM;
        split_mid = btrfs_alloc_extent_map();
        if (!split_mid) {
                ret = -ENOMEM;
                goto out_free_pre;
        }

        btrfs_lock_extent(&inode->io_tree, start, start + len - 1, NULL);
        write_lock(&em_tree->lock);
        em = btrfs_lookup_extent_mapping(em_tree, start, len);
        if (!em) {
                ret = -EIO;
                goto out_unlock;
        }

        ASSERT(em->len == len);
        ASSERT(!btrfs_extent_map_is_compressed(em));
        ASSERT(em->disk_bytenr < EXTENT_MAP_LAST_BYTE);
        ASSERT(em->flags & EXTENT_FLAG_PINNED);
        ASSERT(!(em->flags & EXTENT_FLAG_LOGGING));
        ASSERT(!list_empty(&em->list));

        flags = em->flags;
        em->flags &= ~EXTENT_FLAG_PINNED;

        /* First, replace the em with a new extent_map starting from * em->start */
        split_pre->start = em->start;
        split_pre->len = pre;
        split_pre->disk_bytenr = new_logical;
        split_pre->disk_num_bytes = split_pre->len;
        split_pre->offset = 0;
        split_pre->ram_bytes = split_pre->len;
        split_pre->flags = flags;
        split_pre->generation = em->generation;

        replace_extent_mapping(inode, em, split_pre, 1);

        /*
         * Now we only have an extent_map at:
         *     [em->start, em->start + pre]
         */

        /* Insert the middle extent_map. */
        split_mid->start = em->start + pre;
        split_mid->len = em->len - pre;
        split_mid->disk_bytenr = btrfs_extent_map_block_start(em) + pre;
        split_mid->disk_num_bytes = split_mid->len;
        split_mid->offset = 0;
        split_mid->ram_bytes = split_mid->len;
        split_mid->flags = flags;
        split_mid->generation = em->generation;
        add_extent_mapping(inode, split_mid, 1);

        /* Once for us */
        btrfs_free_extent_map(em);
        /* Once for the tree */
        btrfs_free_extent_map(em);

out_unlock:
        write_unlock(&em_tree->lock);
        btrfs_unlock_extent(&inode->io_tree, start, start + len - 1, NULL);
        btrfs_free_extent_map(split_mid);
out_free_pre:
        btrfs_free_extent_map(split_pre);
        return ret;
}

struct btrfs_em_shrink_ctx {
        long nr_to_scan;
        long scanned;
};

static long btrfs_scan_inode(struct btrfs_inode *inode, struct btrfs_em_shrink_ctx *ctx)
{
        struct btrfs_fs_info *fs_info = inode->root->fs_info;
        const u64 cur_fs_gen = btrfs_get_fs_generation(fs_info);
        struct extent_map_tree *tree = &inode->extent_tree;
        long nr_dropped = 0;
        struct rb_node *node;

        lockdep_assert_held_write(&tree->lock);

        /*
         * Take the mmap lock so that we serialize with the inode logging phase
         * of fsync because we may need to set the full sync flag on the inode,
         * in case we have to remove extent maps in the tree's list of modified
         * extents. If we set the full sync flag in the inode while an fsync is
         * in progress, we may risk missing new extents because before the flag
         * is set, fsync decides to only wait for writeback to complete and then
         * during inode logging it sees the flag set and uses the subvolume tree
         * to find new extents, which may not be there yet because ordered
         * extents haven't completed yet.
         *
         * We also do a try lock because we don't want to block for too long and
         * we are holding the extent map tree's lock in write mode.
         */
        if (!down_read_trylock(&inode->i_mmap_lock))
                return 0;

        node = rb_first(&tree->root);
        while (node) {
                struct rb_node *next = rb_next(node);
                struct extent_map *em;

                em = rb_entry(node, struct extent_map, rb_node);
                ctx->scanned++;

                if (em->flags & EXTENT_FLAG_PINNED)
                        goto next;

                /*
                 * If the inode is in the list of modified extents (new) and its
                 * generation is the same (or is greater than) the current fs
                 * generation, it means it was not yet persisted so we have to
                 * set the full sync flag so that the next fsync will not miss
                 * it.
                 */
                if (!list_empty(&em->list) && em->generation >= cur_fs_gen)
                        btrfs_set_inode_full_sync(inode);

                btrfs_remove_extent_mapping(inode, em);
                trace_btrfs_extent_map_shrinker_remove_em(inode, em);
                /* Drop the reference for the tree. */
                btrfs_free_extent_map(em);
                nr_dropped++;
next:
                if (ctx->scanned >= ctx->nr_to_scan)
                        break;

                /*
                 * Stop if we need to reschedule or there's contention on the
                 * lock. This is to avoid slowing other tasks trying to take the
                 * lock.
                 */
                if (need_resched() || rwlock_needbreak(&tree->lock) ||
                    btrfs_fs_closing(fs_info))
                        break;
                node = next;
        }
        up_read(&inode->i_mmap_lock);

        return nr_dropped;
}

static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root,
                                                      u64 min_ino)
{
        struct btrfs_inode *inode;
        unsigned long from = min_ino;

        xa_lock(&root->inodes);
        while (true) {
                struct extent_map_tree *tree;

                inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT);
                if (!inode)
                        break;

                tree = &inode->extent_tree;

                /*
                 * We want to be fast so if the lock is busy we don't want to
                 * spend time waiting for it (some task is about to do IO for
                 * the inode).
                 */
                if (!write_trylock(&tree->lock))
                        goto next;

                /*
                 * Skip inode if it doesn't have loaded extent maps, so we avoid
                 * getting a reference and doing an iput later. This includes
                 * cases like files that were opened for things like stat(2), or
                 * files with all extent maps previously released through the
                 * release folio callback (btrfs_release_folio()) or released in
                 * a previous run, or directories which never have extent maps.
                 */
                if (RB_EMPTY_ROOT(&tree->root)) {
                        write_unlock(&tree->lock);
                        goto next;
                }

                if (igrab(&inode->vfs_inode))
                        break;

                write_unlock(&tree->lock);
next:
                from = btrfs_ino(inode) + 1;
                cond_resched_lock(&root->inodes.xa_lock);
        }
        xa_unlock(&root->inodes);

        return inode;
}

static long btrfs_scan_root(struct btrfs_root *root, struct btrfs_em_shrink_ctx *ctx)
{
        struct btrfs_fs_info *fs_info = root->fs_info;
        struct btrfs_inode *inode;
        long nr_dropped = 0;
        u64 min_ino = fs_info->em_shrinker_last_ino + 1;

        inode = find_first_inode_to_shrink(root, min_ino);
        while (inode) {
                nr_dropped += btrfs_scan_inode(inode, ctx);
                write_unlock(&inode->extent_tree.lock);

                min_ino = btrfs_ino(inode) + 1;
                fs_info->em_shrinker_last_ino = btrfs_ino(inode);
                iput(&inode->vfs_inode);

                if (ctx->scanned >= ctx->nr_to_scan || btrfs_fs_closing(fs_info))
                        break;

                cond_resched();

                inode = find_first_inode_to_shrink(root, min_ino);
        }

        if (inode) {
                /*
                 * There are still inodes in this root or we happened to process
                 * the last one and reached the scan limit. In either case set
                 * the current root to this one, so we'll resume from the next
                 * inode if there is one or we will find out this was the last
                 * one and move to the next root.
                 */
                fs_info->em_shrinker_last_root = btrfs_root_id(root);
        } else {
                /*
                 * No more inodes in this root, set extent_map_shrinker_last_ino to 0 so
                 * that when processing the next root we start from its first inode.
                 */
                fs_info->em_shrinker_last_ino = 0;
                fs_info->em_shrinker_last_root = btrfs_root_id(root) + 1;
        }

        return nr_dropped;
}

static void btrfs_extent_map_shrinker_worker(struct work_struct *work)
{
        struct btrfs_fs_info *fs_info;
        struct btrfs_em_shrink_ctx ctx;
        u64 start_root_id;
        u64 next_root_id;
        bool cycled = false;
        long nr_dropped = 0;

        fs_info = container_of(work, struct btrfs_fs_info, em_shrinker_work);

        ctx.scanned = 0;
        ctx.nr_to_scan = atomic64_read(&fs_info->em_shrinker_nr_to_scan);

        start_root_id = fs_info->em_shrinker_last_root;
        next_root_id = fs_info->em_shrinker_last_root;

        if (trace_btrfs_extent_map_shrinker_scan_enter_enabled()) {
                s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps);

                trace_btrfs_extent_map_shrinker_scan_enter(fs_info, nr);
        }

        while (ctx.scanned < ctx.nr_to_scan && !btrfs_fs_closing(fs_info)) {
                struct btrfs_root *root;
                unsigned long count;

                cond_resched();

                spin_lock(&fs_info->fs_roots_radix_lock);
                count = radix_tree_gang_lookup(&fs_info->fs_roots_radix,
                                               (void **)&root,
                                               (unsigned long)next_root_id, 1);
                if (count == 0) {
                        spin_unlock(&fs_info->fs_roots_radix_lock);
                        if (start_root_id > 0 && !cycled) {
                                next_root_id = 0;
                                fs_info->em_shrinker_last_root = 0;
                                fs_info->em_shrinker_last_ino = 0;
                                cycled = true;
                                continue;
                        }
                        break;
                }
                next_root_id = btrfs_root_id(root) + 1;
                root = btrfs_grab_root(root);
                spin_unlock(&fs_info->fs_roots_radix_lock);

                if (!root)
                        continue;

                if (is_fstree(btrfs_root_id(root)))
                        nr_dropped += btrfs_scan_root(root, &ctx);

                btrfs_put_root(root);
        }

        if (trace_btrfs_extent_map_shrinker_scan_exit_enabled()) {
                s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps);

                trace_btrfs_extent_map_shrinker_scan_exit(fs_info, nr_dropped, nr);
        }

        atomic64_set(&fs_info->em_shrinker_nr_to_scan, 0);
}

void btrfs_free_extent_maps(struct btrfs_fs_info *fs_info, long nr_to_scan)
{
        /*
         * Do nothing if the shrinker is already running. In case of high memory
         * pressure we can have a lot of tasks calling us and all passing the
         * same nr_to_scan value, but in reality we may need only to free
         * nr_to_scan extent maps (or less). In case we need to free more than
         * that, we will be called again by the fs shrinker, so no worries about
         * not doing enough work to reclaim memory from extent maps.
         * We can also be repeatedly called with the same nr_to_scan value
         * simply because the shrinker runs asynchronously and multiple calls
         * to this function are made before the shrinker does enough progress.
         *
         * That's why we set the atomic counter to nr_to_scan only if its
         * current value is zero, instead of incrementing the counter by
         * nr_to_scan.
         */
        if (atomic64_cmpxchg(&fs_info->em_shrinker_nr_to_scan, 0, nr_to_scan) != 0)
                return;

        queue_work(system_unbound_wq, &fs_info->em_shrinker_work);
}

void btrfs_init_extent_map_shrinker_work(struct btrfs_fs_info *fs_info)
{
        atomic64_set(&fs_info->em_shrinker_nr_to_scan, 0);
        INIT_WORK(&fs_info->em_shrinker_work, btrfs_extent_map_shrinker_worker);
}


























































































    1 














  115 









   94 












  112 












    1 





































   44 


    1 



    1 
    1 

    1 

    1 



    1 
    1 




    1 


    1 


    1 





   44 














   44 


   44 




   44 








   44 
   44 





































   43 

   44 

   44 











   24 








   24 




   24 


   24 





   24 


   23 










   24 



   24 
   24 





   24 









   24 







   24 
   24 
   24 



   24 




   24 


   24 
   24 

   24 

   24 





   43 






   44 


















   43 































  112 
  115 

   49 
  115 
  113 





























    3 








    3 















    3 

    3 

    3 


















   14 
    5 

   16 

    3 
   15 
    6 
   14 

















   15 








   14 
    5 







    2 

   16 






   10 
    6 




   16 


   16 









    1 


    5 





   11 



    2 































































    1 
    1 




    1 


    1 








    1 













    1 






    1 
    1 
    1 








  112 





  113 







  114 

















  114 












  114 





   69 





  114 












  113 


  113 






  114 





  114 





   44 
   70 




  112 

  112 
    1 



  114 










  114 







   70 
   44 


  113 






















    1 


















    2 
   71 
    1 
   70 



   11 
    1 





    1 


   11 
   11 













   70 







  114 




  114 


  114 





















   71 








    1 
   70 



   71 

   71 
    2 





    1 



   71 
   13 







    3 




   10 








   11 






   11 
    1 














   10 












   70 







   69 











   70 







   69 








   70 





   70 


















   70 





   70 




   70 




   70 










   70 
   69 









   70 


    1 








   70 
   70 




   70 






   68 
   69 









   70 


   70 








   70 
   70 









    1 


   70 
   70 


    6 








   69 






   70 












   70 







   69 



















   70 



   69 













   70 

   70 
    1 


   70 

   70 


    5 
   69 






    5 







   70 










   70 



   70 
   70 
   70 

   69 





   70 











   69 



   70 








































   70 








   70 




    7 






   70 
    7 





    7 
   70 



    2 


    9 
   70 
    4 

   14 




   70 
    4 
























































































































































































































































































































   70 







   70 

   70 





































   70 













   70 






   70 


   70 



























   70 







   70 









    7 







   68 









   70 
   70 



































































































































































































































































































































































   44 














   44 






















   44 



























































   44 













   44 



   44 









   44 


    1 



   44 













   42 



   44 



    1 






   44 
   44 
   44 







   44 

   44 


   44 





    1 

   44 










   44 









   44 



   44 












   44 














































   44 




   44 


   44 
   44 


















   45 























   24 



   44 


   44 
   45 

   45 





   45 



   44 

   45 




































   44 


   44 








   44 


   42 
    1 









    1 






    1 
    1 






    1 










    1 

























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
// SPDX-License-Identifier: GPL-2.0

#include "bcachefs.h"
#include "async_objs.h"
#include "bkey_buf.h"
#include "bkey_methods.h"
#include "bkey_sort.h"
#include "btree_cache.h"
#include "btree_io.h"
#include "btree_iter.h"
#include "btree_locking.h"
#include "btree_update.h"
#include "btree_update_interior.h"
#include "buckets.h"
#include "checksum.h"
#include "debug.h"
#include "enumerated_ref.h"
#include "error.h"
#include "extents.h"
#include "io_write.h"
#include "journal_reclaim.h"
#include "journal_seq_blacklist.h"
#include "recovery.h"
#include "super-io.h"
#include "trace.h"

#include <linux/sched/mm.h>

static void bch2_btree_node_header_to_text(struct printbuf *out, struct btree_node *bn)
{
        bch2_btree_id_level_to_text(out, BTREE_NODE_ID(bn), BTREE_NODE_LEVEL(bn));
        prt_printf(out, " seq %llx %llu\n", bn->keys.seq, BTREE_NODE_SEQ(bn));
        prt_str(out, "min: ");
        bch2_bpos_to_text(out, bn->min_key);
        prt_newline(out);
        prt_str(out, "max: ");
        bch2_bpos_to_text(out, bn->max_key);
}

void bch2_btree_node_io_unlock(struct btree *b)
{
        EBUG_ON(!btree_node_write_in_flight(b));

        clear_btree_node_write_in_flight_inner(b);
        clear_btree_node_write_in_flight(b);
        smp_mb__after_atomic();
        wake_up_bit(&b->flags, BTREE_NODE_write_in_flight);
}

void bch2_btree_node_io_lock(struct btree *b)
{
        wait_on_bit_lock_io(&b->flags, BTREE_NODE_write_in_flight,
                            TASK_UNINTERRUPTIBLE);
}

void __bch2_btree_node_wait_on_read(struct btree *b)
{
        wait_on_bit_io(&b->flags, BTREE_NODE_read_in_flight,
                       TASK_UNINTERRUPTIBLE);
}

void __bch2_btree_node_wait_on_write(struct btree *b)
{
        wait_on_bit_io(&b->flags, BTREE_NODE_write_in_flight,
                       TASK_UNINTERRUPTIBLE);
}

void bch2_btree_node_wait_on_read(struct btree *b)
{
        wait_on_bit_io(&b->flags, BTREE_NODE_read_in_flight,
                       TASK_UNINTERRUPTIBLE);
}

void bch2_btree_node_wait_on_write(struct btree *b)
{
        wait_on_bit_io(&b->flags, BTREE_NODE_write_in_flight,
                       TASK_UNINTERRUPTIBLE);
}

static void verify_no_dups(struct btree *b,
                           struct bkey_packed *start,
                           struct bkey_packed *end)
{
#ifdef CONFIG_BCACHEFS_DEBUG
        struct bkey_packed *k, *p;

        if (start == end)
                return;

        for (p = start, k = bkey_p_next(start);
             k != end;
             p = k, k = bkey_p_next(k)) {
                struct bkey l = bkey_unpack_key(b, p);
                struct bkey r = bkey_unpack_key(b, k);

                BUG_ON(bpos_ge(l.p, bkey_start_pos(&r)));
        }
#endif
}

static void set_needs_whiteout(struct bset *i, int v)
{
        struct bkey_packed *k;

        for (k = i->start; k != vstruct_last(i); k = bkey_p_next(k))
                k->needs_whiteout = v;
}

static void btree_bounce_free(struct bch_fs *c, size_t size,
                              bool used_mempool, void *p)
{
        if (used_mempool)
                mempool_free(p, &c->btree_bounce_pool);
        else
                kvfree(p);
}

static void *btree_bounce_alloc(struct bch_fs *c, size_t size,
                                bool *used_mempool)
{
        unsigned flags = memalloc_nofs_save();
        void *p;

        BUG_ON(size > c->opts.btree_node_size);

        *used_mempool = false;
        p = kvmalloc(size, __GFP_NOWARN|GFP_NOWAIT);
        if (!p) {
                *used_mempool = true;
                p = mempool_alloc(&c->btree_bounce_pool, GFP_NOFS);
        }
        memalloc_nofs_restore(flags);
        return p;
}

static void sort_bkey_ptrs(const struct btree *bt,
                           struct bkey_packed **ptrs, unsigned nr)
{
        unsigned n = nr, a = nr / 2, b, c, d;

        if (!a)
                return;

        /* Heap sort: see lib/sort.c: */
        while (1) {
                if (a)
                        a--;
                else if (--n)
                        swap(ptrs[0], ptrs[n]);
                else
                        break;

                for (b = a; c = 2 * b + 1, (d = c + 1) < n;)
                        b = bch2_bkey_cmp_packed(bt,
                                            ptrs[c],
                                            ptrs[d]) >= 0 ? c : d;
                if (d == n)
                        b = c;

                while (b != a &&
                       bch2_bkey_cmp_packed(bt,
                                       ptrs[a],
                                       ptrs[b]) >= 0)
                        b = (b - 1) / 2;
                c = b;
                while (b != a) {
                        b = (b - 1) / 2;
                        swap(ptrs[b], ptrs[c]);
                }
        }
}

static void bch2_sort_whiteouts(struct bch_fs *c, struct btree *b)
{
        struct bkey_packed *new_whiteouts, **ptrs, **ptrs_end, *k;
        bool used_mempool = false;
        size_t bytes = b->whiteout_u64s * sizeof(u64);

        if (!b->whiteout_u64s)
                return;

        new_whiteouts = btree_bounce_alloc(c, bytes, &used_mempool);

        ptrs = ptrs_end = ((void *) new_whiteouts + bytes);

        for (k = unwritten_whiteouts_start(b);
             k != unwritten_whiteouts_end(b);
             k = bkey_p_next(k))
                *--ptrs = k;

        sort_bkey_ptrs(b, ptrs, ptrs_end - ptrs);

        k = new_whiteouts;

        while (ptrs != ptrs_end) {
                bkey_p_copy(k, *ptrs);
                k = bkey_p_next(k);
                ptrs++;
        }

        verify_no_dups(b, new_whiteouts,
                       (void *) ((u64 *) new_whiteouts + b->whiteout_u64s));

        memcpy_u64s(unwritten_whiteouts_start(b),
                    new_whiteouts, b->whiteout_u64s);

        btree_bounce_free(c, bytes, used_mempool, new_whiteouts);
}

static bool should_compact_bset(struct btree *b, struct bset_tree *t,
                                bool compacting, enum compact_mode mode)
{
        if (!bset_dead_u64s(b, t))
                return false;

        switch (mode) {
        case COMPACT_LAZY:
                return should_compact_bset_lazy(b, t) ||
                        (compacting && !bset_written(b, bset(b, t)));
        case COMPACT_ALL:
                return true;
        default:
                BUG();
        }
}

static bool bch2_drop_whiteouts(struct btree *b, enum compact_mode mode)
{
        bool ret = false;

        for_each_bset(b, t) {
                struct bset *i = bset(b, t);
                struct bkey_packed *k, *n, *out, *start, *end;
                struct btree_node_entry *src = NULL, *dst = NULL;

                if (t != b->set && !bset_written(b, i)) {
                        src = container_of(i, struct btree_node_entry, keys);
                        dst = max(write_block(b),
                                  (void *) btree_bkey_last(b, t - 1));
                }

                if (src != dst)
                        ret = true;

                if (!should_compact_bset(b, t, ret, mode)) {
                        if (src != dst) {
                                memmove(dst, src, sizeof(*src) +
                                        le16_to_cpu(src->keys.u64s) *
                                        sizeof(u64));
                                i = &dst->keys;
                                set_btree_bset(b, t, i);
                        }
                        continue;
                }

                start        = btree_bkey_first(b, t);
                end        = btree_bkey_last(b, t);

                if (src != dst) {
                        memmove(dst, src, sizeof(*src));
                        i = &dst->keys;
                        set_btree_bset(b, t, i);
                }

                out = i->start;

                for (k = start; k != end; k = n) {
                        n = bkey_p_next(k);

                        if (!bkey_deleted(k)) {
                                bkey_p_copy(out, k);
                                out = bkey_p_next(out);
                        } else {
                                BUG_ON(k->needs_whiteout);
                        }
                }

                i->u64s = cpu_to_le16((u64 *) out - i->_data);
                set_btree_bset_end(b, t);
                bch2_bset_set_no_aux_tree(b, t);
                ret = true;
        }

        bch2_verify_btree_nr_keys(b);

        bch2_btree_build_aux_trees(b);

        return ret;
}

bool bch2_compact_whiteouts(struct bch_fs *c, struct btree *b,
                            enum compact_mode mode)
{
        return bch2_drop_whiteouts(b, mode);
}

static void btree_node_sort(struct bch_fs *c, struct btree *b,
                            unsigned start_idx,
                            unsigned end_idx)
{
        struct btree_node *out;
        struct sort_iter_stack sort_iter;
        struct bset_tree *t;
        struct bset *start_bset = bset(b, &b->set[start_idx]);
        bool used_mempool = false;
        u64 start_time, seq = 0;
        unsigned i, u64s = 0, bytes, shift = end_idx - start_idx - 1;
        bool sorting_entire_node = start_idx == 0 &&
                end_idx == b->nsets;

        sort_iter_stack_init(&sort_iter, b);

        for (t = b->set + start_idx;
             t < b->set + end_idx;
             t++) {
                u64s += le16_to_cpu(bset(b, t)->u64s);
                sort_iter_add(&sort_iter.iter,
                              btree_bkey_first(b, t),
                              btree_bkey_last(b, t));
        }

        bytes = sorting_entire_node
                ? btree_buf_bytes(b)
                : __vstruct_bytes(struct btree_node, u64s);

        out = btree_bounce_alloc(c, bytes, &used_mempool);

        start_time = local_clock();

        u64s = bch2_sort_keys(out->keys.start, &sort_iter.iter);

        out->keys.u64s = cpu_to_le16(u64s);

        BUG_ON(vstruct_end(&out->keys) > (void *) out + bytes);

        if (sorting_entire_node)
                bch2_time_stats_update(&c->times[BCH_TIME_btree_node_sort],
                                       start_time);

        /* Make sure we preserve bset journal_seq: */
        for (t = b->set + start_idx; t < b->set + end_idx; t++)
                seq = max(seq, le64_to_cpu(bset(b, t)->journal_seq));
        start_bset->journal_seq = cpu_to_le64(seq);

        if (sorting_entire_node) {
                u64s = le16_to_cpu(out->keys.u64s);

                BUG_ON(bytes != btree_buf_bytes(b));

                /*
                 * Our temporary buffer is the same size as the btree node's
                 * buffer, we can just swap buffers instead of doing a big
                 * memcpy()
                 */
                *out = *b->data;
                out->keys.u64s = cpu_to_le16(u64s);
                swap(out, b->data);
                set_btree_bset(b, b->set, &b->data->keys);
        } else {
                start_bset->u64s = out->keys.u64s;
                memcpy_u64s(start_bset->start,
                            out->keys.start,
                            le16_to_cpu(out->keys.u64s));
        }

        for (i = start_idx + 1; i < end_idx; i++)
                b->nr.bset_u64s[start_idx] +=
                        b->nr.bset_u64s[i];

        b->nsets -= shift;

        for (i = start_idx + 1; i < b->nsets; i++) {
                b->nr.bset_u64s[i]        = b->nr.bset_u64s[i + shift];
                b->set[i]                = b->set[i + shift];
        }

        for (i = b->nsets; i < MAX_BSETS; i++)
                b->nr.bset_u64s[i] = 0;

        set_btree_bset_end(b, &b->set[start_idx]);
        bch2_bset_set_no_aux_tree(b, &b->set[start_idx]);

        btree_bounce_free(c, bytes, used_mempool, out);

        bch2_verify_btree_nr_keys(b);
}

void bch2_btree_sort_into(struct bch_fs *c,
                         struct btree *dst,
                         struct btree *src)
{
        struct btree_nr_keys nr;
        struct btree_node_iter src_iter;
        u64 start_time = local_clock();

        BUG_ON(dst->nsets != 1);

        bch2_bset_set_no_aux_tree(dst, dst->set);

        bch2_btree_node_iter_init_from_start(&src_iter, src);

        nr = bch2_sort_repack(btree_bset_first(dst),
                        src, &src_iter,
                        &dst->format,
                        true);

        bch2_time_stats_update(&c->times[BCH_TIME_btree_node_sort],
                               start_time);

        set_btree_bset_end(dst, dst->set);

        dst->nr.live_u64s        += nr.live_u64s;
        dst->nr.bset_u64s[0]        += nr.bset_u64s[0];
        dst->nr.packed_keys        += nr.packed_keys;
        dst->nr.unpacked_keys        += nr.unpacked_keys;

        bch2_verify_btree_nr_keys(dst);
}

/*
 * We're about to add another bset to the btree node, so if there's currently
 * too many bsets - sort some of them together:
 */
static bool btree_node_compact(struct bch_fs *c, struct btree *b)
{
        unsigned unwritten_idx;
        bool ret = false;

        for (unwritten_idx = 0;
             unwritten_idx < b->nsets;
             unwritten_idx++)
                if (!bset_written(b, bset(b, &b->set[unwritten_idx])))
                        break;

        if (b->nsets - unwritten_idx > 1) {
                btree_node_sort(c, b, unwritten_idx, b->nsets);
                ret = true;
        }

        if (unwritten_idx > 1) {
                btree_node_sort(c, b, 0, unwritten_idx);
                ret = true;
        }

        return ret;
}

void bch2_btree_build_aux_trees(struct btree *b)
{
        for_each_bset(b, t)
                bch2_bset_build_aux_tree(b, t,
                                !bset_written(b, bset(b, t)) &&
                                t == bset_tree_last(b));
}

/*
 * If we have MAX_BSETS (3) bsets, should we sort them all down to just one?
 *
 * The first bset is going to be of similar order to the size of the node, the
 * last bset is bounded by btree_write_set_buffer(), which is set to keep the
 * memmove on insert from being too expensive: the middle bset should, ideally,
 * be the geometric mean of the first and the last.
 *
 * Returns true if the middle bset is greater than that geometric mean:
 */
static inline bool should_compact_all(struct bch_fs *c, struct btree *b)
{
        unsigned mid_u64s_bits =
                (ilog2(btree_max_u64s(c)) + BTREE_WRITE_SET_U64s_BITS) / 2;

        return bset_u64s(&b->set[1]) > 1U << mid_u64s_bits;
}

/*
 * @bch_btree_init_next - initialize a new (unwritten) bset that can then be
 * inserted into
 *
 * Safe to call if there already is an unwritten bset - will only add a new bset
 * if @b doesn't already have one.
 *
 * Returns true if we sorted (i.e. invalidated iterators
 */
void bch2_btree_init_next(struct btree_trans *trans, struct btree *b)
{
        struct bch_fs *c = trans->c;
        struct btree_node_entry *bne;
        bool reinit_iter = false;

        EBUG_ON(!six_lock_counts(&b->c.lock).n[SIX_LOCK_write]);
        BUG_ON(bset_written(b, bset(b, &b->set[1])));
        BUG_ON(btree_node_just_written(b));

        if (b->nsets == MAX_BSETS &&
            !btree_node_write_in_flight(b) &&
            should_compact_all(c, b)) {
                bch2_btree_node_write_trans(trans, b, SIX_LOCK_write,
                                            BTREE_WRITE_init_next_bset);
                reinit_iter = true;
        }

        if (b->nsets == MAX_BSETS &&
            btree_node_compact(c, b))
                reinit_iter = true;

        BUG_ON(b->nsets >= MAX_BSETS);

        bne = want_new_bset(c, b);
        if (bne)
                bch2_bset_init_next(b, bne);

        bch2_btree_build_aux_trees(b);

        if (reinit_iter)
                bch2_trans_node_reinit_iter(trans, b);
}

static void btree_err_msg(struct printbuf *out, struct bch_fs *c,
                          struct bch_dev *ca,
                          bool print_pos,
                          struct btree *b, struct bset *i, struct bkey_packed *k,
                          unsigned offset, int rw)
{
        if (print_pos) {
                prt_str(out, rw == READ
                        ? "error validating btree node "
                        : "corrupt btree node before write ");
                prt_printf(out, "at btree ");
                bch2_btree_pos_to_text(out, c, b);
                prt_newline(out);
        }

        if (ca)
                prt_printf(out, "%s ", ca->name);

        prt_printf(out, "node offset %u/%u",
                   b->written, btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)));
        if (i)
                prt_printf(out, " bset u64s %u", le16_to_cpu(i->u64s));
        if (k)
                prt_printf(out, " bset byte offset %lu",
                           (unsigned long)(void *)k -
                           ((unsigned long)(void *)i & ~511UL));
        prt_str(out, ": ");
}

__printf(11, 12)
static int __btree_err(int ret,
                       struct bch_fs *c,
                       struct bch_dev *ca,
                       struct btree *b,
                       struct bset *i,
                       struct bkey_packed *k,
                       int rw,
                       enum bch_sb_error_id err_type,
                       struct bch_io_failures *failed,
                       struct printbuf *err_msg,
                       const char *fmt, ...)
{
        if (c->recovery.curr_pass == BCH_RECOVERY_PASS_scan_for_btree_nodes)
                return ret == -BCH_ERR_btree_node_read_err_fixable
                        ? bch_err_throw(c, fsck_fix)
                        : ret;

        bool have_retry = false;
        int ret2;

        if (ca) {
                bch2_mark_btree_validate_failure(failed, ca->dev_idx);

                struct extent_ptr_decoded pick;
                have_retry = !bch2_bkey_pick_read_device(c,
                                        bkey_i_to_s_c(&b->key),
                                        failed, &pick, -1);
        }

        if (!have_retry && ret == -BCH_ERR_btree_node_read_err_want_retry)
                ret = bch_err_throw(c, btree_node_read_err_fixable);
        if (!have_retry && ret == -BCH_ERR_btree_node_read_err_must_retry)
                ret = bch_err_throw(c, btree_node_read_err_bad_node);

        bch2_sb_error_count(c, err_type);

        bool print_deferred = err_msg &&
                rw == READ &&
                !(test_bit(BCH_FS_in_fsck, &c->flags) &&
                  c->opts.fix_errors == FSCK_FIX_ask);

        struct printbuf out = PRINTBUF;
        bch2_log_msg_start(c, &out);

        if (!print_deferred)
                err_msg = &out;

        btree_err_msg(err_msg, c, ca, !print_deferred, b, i, k, b->written, rw);

        va_list args;
        va_start(args, fmt);
        prt_vprintf(err_msg, fmt, args);
        va_end(args);

        if (print_deferred) {
                prt_newline(err_msg);

                switch (ret) {
                case -BCH_ERR_btree_node_read_err_fixable:
                        ret2 = bch2_fsck_err_opt(c, FSCK_CAN_FIX, err_type);
                        if (!bch2_err_matches(ret2, BCH_ERR_fsck_fix) &&
                            !bch2_err_matches(ret2, BCH_ERR_fsck_ignore)) {
                                ret = ret2;
                                goto fsck_err;
                        }

                        if (!have_retry)
                                ret = bch_err_throw(c, fsck_fix);
                        goto out;
                case -BCH_ERR_btree_node_read_err_bad_node:
                        prt_str(&out, ", ");
                        ret = __bch2_topology_error(c, &out);
                        break;
                }

                goto out;
        }

        if (rw == WRITE) {
                prt_str(&out, ", ");
                ret = __bch2_inconsistent_error(c, &out)
                        ? -BCH_ERR_fsck_errors_not_fixed
                        : 0;
                goto print;
        }

        switch (ret) {
        case -BCH_ERR_btree_node_read_err_fixable:
                ret2 = __bch2_fsck_err(c, NULL, FSCK_CAN_FIX, err_type, "%s", out.buf);
                if (!bch2_err_matches(ret2, BCH_ERR_fsck_fix) &&
                    !bch2_err_matches(ret2, BCH_ERR_fsck_ignore)) {
                        ret = ret2;
                        goto fsck_err;
                }

                if (!have_retry)
                        ret = bch_err_throw(c, fsck_fix);
                goto out;
        case -BCH_ERR_btree_node_read_err_bad_node:
                prt_str(&out, ", ");
                ret = __bch2_topology_error(c, &out);
                break;
        }
print:
        bch2_print_str(c, KERN_ERR, out.buf);
out:
fsck_err:
        printbuf_exit(&out);
        return ret;
}

#define btree_err(type, c, ca, b, i, k, _err_type, msg, ...)                \
({                                                                        \
        int _ret = __btree_err(type, c, ca, b, i, k, write,                \
                               BCH_FSCK_ERR_##_err_type,                \
                               failed, err_msg,                                \
                               msg, ##__VA_ARGS__);                        \
                                                                        \
        if (!bch2_err_matches(_ret, BCH_ERR_fsck_fix)) {                \
                ret = _ret;                                                \
                goto fsck_err;                                                \
        }                                                                \
                                                                        \
        true;                                                                \
})

#define btree_err_on(cond, ...)        ((cond) ? btree_err(__VA_ARGS__) : false)

/*
 * When btree topology repair changes the start or end of a node, that might
 * mean we have to drop keys that are no longer inside the node:
 */
__cold
void bch2_btree_node_drop_keys_outside_node(struct btree *b)
{
        for_each_bset(b, t) {
                struct bset *i = bset(b, t);
                struct bkey_packed *k;

                for (k = i->start; k != vstruct_last(i); k = bkey_p_next(k))
                        if (bkey_cmp_left_packed(b, k, &b->data->min_key) >= 0)
                                break;

                if (k != i->start) {
                        unsigned shift = (u64 *) k - (u64 *) i->start;

                        memmove_u64s_down(i->start, k,
                                          (u64 *) vstruct_end(i) - (u64 *) k);
                        i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - shift);
                        set_btree_bset_end(b, t);
                }

                for (k = i->start; k != vstruct_last(i); k = bkey_p_next(k))
                        if (bkey_cmp_left_packed(b, k, &b->data->max_key) > 0)
                                break;

                if (k != vstruct_last(i)) {
                        i->u64s = cpu_to_le16((u64 *) k - (u64 *) i->start);
                        set_btree_bset_end(b, t);
                }
        }

        /*
         * Always rebuild search trees: eytzinger search tree nodes directly
         * depend on the values of min/max key:
         */
        bch2_bset_set_no_aux_tree(b, b->set);
        bch2_btree_build_aux_trees(b);
        b->nr = bch2_btree_node_count_keys(b);

        struct bkey_s_c k;
        struct bkey unpacked;
        struct btree_node_iter iter;
        for_each_btree_node_key_unpack(b, k, &iter, &unpacked) {
                BUG_ON(bpos_lt(k.k->p, b->data->min_key));
                BUG_ON(bpos_gt(k.k->p, b->data->max_key));
        }
}

static int validate_bset(struct bch_fs *c, struct bch_dev *ca,
                         struct btree *b, struct bset *i,
                         unsigned offset, int write,
                         struct bch_io_failures *failed,
                         struct printbuf *err_msg)
{
        unsigned version = le16_to_cpu(i->version);
        struct printbuf buf1 = PRINTBUF;
        struct printbuf buf2 = PRINTBUF;
        int ret = 0;

        btree_err_on(!bch2_version_compatible(version),
                     -BCH_ERR_btree_node_read_err_incompatible,
                     c, ca, b, i, NULL,
                     btree_node_unsupported_version,
                     "unsupported bset version %u.%u",
                     BCH_VERSION_MAJOR(version),
                     BCH_VERSION_MINOR(version));

        if (c->recovery.curr_pass != BCH_RECOVERY_PASS_scan_for_btree_nodes &&
            btree_err_on(version < c->sb.version_min,
                         -BCH_ERR_btree_node_read_err_fixable,
                         c, NULL, b, i, NULL,
                         btree_node_bset_older_than_sb_min,
                         "bset version %u older than superblock version_min %u",
                         version, c->sb.version_min)) {
                if (bch2_version_compatible(version)) {
                        mutex_lock(&c->sb_lock);
                        c->disk_sb.sb->version_min = cpu_to_le16(version);
                        bch2_write_super(c);
                        mutex_unlock(&c->sb_lock);
                } else {
                        /* We have no idea what's going on: */
                        i->version = cpu_to_le16(c->sb.version);
                }
        }

        if (btree_err_on(BCH_VERSION_MAJOR(version) >
                         BCH_VERSION_MAJOR(c->sb.version),
                         -BCH_ERR_btree_node_read_err_fixable,
                         c, NULL, b, i, NULL,
                         btree_node_bset_newer_than_sb,
                         "bset version %u newer than superblock version %u",
                         version, c->sb.version)) {
                mutex_lock(&c->sb_lock);
                c->disk_sb.sb->version = cpu_to_le16(version);
                bch2_write_super(c);
                mutex_unlock(&c->sb_lock);
        }

        btree_err_on(BSET_SEPARATE_WHITEOUTS(i),
                     -BCH_ERR_btree_node_read_err_incompatible,
                     c, ca, b, i, NULL,
                     btree_node_unsupported_version,
                     "BSET_SEPARATE_WHITEOUTS no longer supported");

        btree_err_on(offset && !i->u64s,
                     -BCH_ERR_btree_node_read_err_fixable,
                     c, ca, b, i, NULL,
                     bset_empty,
                     "empty bset");

        btree_err_on(BSET_OFFSET(i) && BSET_OFFSET(i) != offset,
                     -BCH_ERR_btree_node_read_err_want_retry,
                     c, ca, b, i, NULL,
                     bset_wrong_sector_offset,
                     "bset at wrong sector offset");

        if (!offset) {
                struct btree_node *bn =
                        container_of(i, struct btree_node, keys);
                /* These indicate that we read the wrong btree node: */

                if (b->key.k.type == KEY_TYPE_btree_ptr_v2) {
                        struct bch_btree_ptr_v2 *bp =
                                &bkey_i_to_btree_ptr_v2(&b->key)->v;

                        /* XXX endianness */
                        btree_err_on(bp->seq != bn->keys.seq,
                                     -BCH_ERR_btree_node_read_err_must_retry,
                                     c, ca, b, NULL, NULL,
                                     bset_bad_seq,
                                     "incorrect sequence number (wrong btree node)");
                }

                btree_err_on(BTREE_NODE_ID(bn) != b->c.btree_id,
                             -BCH_ERR_btree_node_read_err_must_retry,
                             c, ca, b, i, NULL,
                             btree_node_bad_btree,
                             "incorrect btree id");

                btree_err_on(BTREE_NODE_LEVEL(bn) != b->c.level,
                             -BCH_ERR_btree_node_read_err_must_retry,
                             c, ca, b, i, NULL,
                             btree_node_bad_level,
                             "incorrect level");

                if (!write)
                        compat_btree_node(b->c.level, b->c.btree_id, version,
                                          BSET_BIG_ENDIAN(i), write, bn);

                if (b->key.k.type == KEY_TYPE_btree_ptr_v2) {
                        struct bch_btree_ptr_v2 *bp =
                                &bkey_i_to_btree_ptr_v2(&b->key)->v;

                        if (BTREE_PTR_RANGE_UPDATED(bp)) {
                                b->data->min_key = bp->min_key;
                                b->data->max_key = b->key.k.p;
                        }

                        btree_err_on(!bpos_eq(b->data->min_key, bp->min_key),
                                     -BCH_ERR_btree_node_read_err_must_retry,
                                     c, ca, b, NULL, NULL,
                                     btree_node_bad_min_key,
                                     "incorrect min_key: got %s should be %s",
                                     (printbuf_reset(&buf1),
                                      bch2_bpos_to_text(&buf1, bn->min_key), buf1.buf),
                                     (printbuf_reset(&buf2),
                                      bch2_bpos_to_text(&buf2, bp->min_key), buf2.buf));
                }

                btree_err_on(!bpos_eq(bn->max_key, b->key.k.p),
                             -BCH_ERR_btree_node_read_err_must_retry,
                             c, ca, b, i, NULL,
                             btree_node_bad_max_key,
                             "incorrect max key %s",
                             (printbuf_reset(&buf1),
                              bch2_bpos_to_text(&buf1, bn->max_key), buf1.buf));

                if (write)
                        compat_btree_node(b->c.level, b->c.btree_id, version,
                                          BSET_BIG_ENDIAN(i), write, bn);

                btree_err_on(bch2_bkey_format_invalid(c, &bn->format, write, &buf1),
                             -BCH_ERR_btree_node_read_err_bad_node,
                             c, ca, b, i, NULL,
                             btree_node_bad_format,
                             "invalid bkey format: %s\n%s", buf1.buf,
                             (printbuf_reset(&buf2),
                              bch2_bkey_format_to_text(&buf2, &bn->format), buf2.buf));
                printbuf_reset(&buf1);

                compat_bformat(b->c.level, b->c.btree_id, version,
                               BSET_BIG_ENDIAN(i), write,
                               &bn->format);
        }
fsck_err:
        printbuf_exit(&buf2);
        printbuf_exit(&buf1);
        return ret;
}

static int btree_node_bkey_val_validate(struct bch_fs *c, struct btree *b,
                                        struct bkey_s_c k,
                                        enum bch_validate_flags flags)
{
        return bch2_bkey_val_validate(c, k, (struct bkey_validate_context) {
                .from        = BKEY_VALIDATE_btree_node,
                .level        = b->c.level,
                .btree        = b->c.btree_id,
                .flags        = flags
        });
}

static int bset_key_validate(struct bch_fs *c, struct btree *b,
                             struct bkey_s_c k,
                             bool updated_range,
                             enum bch_validate_flags flags)
{
        struct bkey_validate_context from = (struct bkey_validate_context) {
                .from        = BKEY_VALIDATE_btree_node,
                .level        = b->c.level,
                .btree        = b->c.btree_id,
                .flags        = flags,
        };
        return __bch2_bkey_validate(c, k, from) ?:
                (!updated_range ? bch2_bkey_in_btree_node(c, b, k, from) : 0) ?:
                (flags & BCH_VALIDATE_write ? btree_node_bkey_val_validate(c, b, k, flags) : 0);
}

static bool bkey_packed_valid(struct bch_fs *c, struct btree *b,
                         struct bset *i, struct bkey_packed *k)
{
        if (bkey_p_next(k) > vstruct_last(i))
                return false;

        if (k->format > KEY_FORMAT_CURRENT)
                return false;

        if (!bkeyp_u64s_valid(&b->format, k))
                return false;

        struct bkey tmp;
        struct bkey_s u = __bkey_disassemble(b, k, &tmp);
        return !__bch2_bkey_validate(c, u.s_c,
                                     (struct bkey_validate_context) {
                                        .from        = BKEY_VALIDATE_btree_node,
                                        .level        = b->c.level,
                                        .btree        = b->c.btree_id,
                                        .flags        = BCH_VALIDATE_silent
                                     });
}

static inline int btree_node_read_bkey_cmp(const struct btree *b,
                                const struct bkey_packed *l,
                                const struct bkey_packed *r)
{
        return bch2_bkey_cmp_packed(b, l, r)
                ?: (int) bkey_deleted(r) - (int) bkey_deleted(l);
}

static int validate_bset_keys(struct bch_fs *c, struct btree *b,
                         struct bset *i, int write,
                         struct bch_io_failures *failed,
                         struct printbuf *err_msg)
{
        unsigned version = le16_to_cpu(i->version);
        struct bkey_packed *k, *prev = NULL;
        struct printbuf buf = PRINTBUF;
        bool updated_range = b->key.k.type == KEY_TYPE_btree_ptr_v2 &&
                BTREE_PTR_RANGE_UPDATED(&bkey_i_to_btree_ptr_v2(&b->key)->v);
        int ret = 0;

        for (k = i->start;
             k != vstruct_last(i);) {
                struct bkey_s u;
                struct bkey tmp;
                unsigned next_good_key;

                if (btree_err_on(bkey_p_next(k) > vstruct_last(i),
                                 -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, i, k,
                                 btree_node_bkey_past_bset_end,
                                 "key extends past end of bset")) {
                        i->u64s = cpu_to_le16((u64 *) k - i->_data);
                        break;
                }

                if (btree_err_on(k->format > KEY_FORMAT_CURRENT,
                                 -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, i, k,
                                 btree_node_bkey_bad_format,
                                 "invalid bkey format %u", k->format))
                        goto drop_this_key;

                if (btree_err_on(!bkeyp_u64s_valid(&b->format, k),
                                 -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, i, k,
                                 btree_node_bkey_bad_u64s,
                                 "bad k->u64s %u (min %u max %zu)", k->u64s,
                                 bkeyp_key_u64s(&b->format, k),
                                 U8_MAX - BKEY_U64s + bkeyp_key_u64s(&b->format, k)))
                        goto drop_this_key;

                if (!write)
                        bch2_bkey_compat(b->c.level, b->c.btree_id, version,
                                    BSET_BIG_ENDIAN(i), write,
                                    &b->format, k);

                u = __bkey_disassemble(b, k, &tmp);

                ret = bset_key_validate(c, b, u.s_c, updated_range, write);
                if (ret == -BCH_ERR_fsck_delete_bkey)
                        goto drop_this_key;
                if (ret)
                        goto fsck_err;

                if (write)
                        bch2_bkey_compat(b->c.level, b->c.btree_id, version,
                                    BSET_BIG_ENDIAN(i), write,
                                    &b->format, k);

                if (prev && btree_node_read_bkey_cmp(b, prev, k) >= 0) {
                        struct bkey up = bkey_unpack_key(b, prev);

                        printbuf_reset(&buf);
                        prt_printf(&buf, "keys out of order: ");
                        bch2_bkey_to_text(&buf, &up);
                        prt_printf(&buf, " > ");
                        bch2_bkey_to_text(&buf, u.k);

                        if (btree_err(-BCH_ERR_btree_node_read_err_fixable,
                                      c, NULL, b, i, k,
                                      btree_node_bkey_out_of_order,
                                      "%s", buf.buf))
                                goto drop_this_key;
                }

                prev = k;
                k = bkey_p_next(k);
                continue;
drop_this_key:
                next_good_key = k->u64s;

                if (!next_good_key ||
                    (BSET_BIG_ENDIAN(i) == CPU_BIG_ENDIAN &&
                     version >= bcachefs_metadata_version_snapshot)) {
                        /*
                         * only do scanning if bch2_bkey_compat() has nothing to
                         * do
                         */

                        if (!bkey_packed_valid(c, b, i, (void *) ((u64 *) k + next_good_key))) {
                                for (next_good_key = 1;
                                     next_good_key < (u64 *) vstruct_last(i) - (u64 *) k;
                                     next_good_key++)
                                        if (bkey_packed_valid(c, b, i, (void *) ((u64 *) k + next_good_key)))
                                                goto got_good_key;
                        }

                        /*
                         * didn't find a good key, have to truncate the rest of
                         * the bset
                         */
                        next_good_key = (u64 *) vstruct_last(i) - (u64 *) k;
                }
got_good_key:
                le16_add_cpu(&i->u64s, -next_good_key);
                memmove_u64s_down(k, (u64 *) k + next_good_key, (u64 *) vstruct_end(i) - (u64 *) k);
                set_btree_node_need_rewrite(b);
                set_btree_node_need_rewrite_error(b);
        }
fsck_err:
        printbuf_exit(&buf);
        return ret;
}

int bch2_btree_node_read_done(struct bch_fs *c, struct bch_dev *ca,
                              struct btree *b,
                              struct bch_io_failures *failed,
                              struct printbuf *err_msg)
{
        struct btree_node_entry *bne;
        struct sort_iter *iter;
        struct btree_node *sorted;
        struct bkey_packed *k;
        struct bset *i;
        bool used_mempool, blacklisted;
        bool updated_range = b->key.k.type == KEY_TYPE_btree_ptr_v2 &&
                BTREE_PTR_RANGE_UPDATED(&bkey_i_to_btree_ptr_v2(&b->key)->v);
        unsigned ptr_written = btree_ptr_sectors_written(bkey_i_to_s_c(&b->key));
        u64 max_journal_seq = 0;
        struct printbuf buf = PRINTBUF;
        int ret = 0, write = READ;
        u64 start_time = local_clock();

        b->version_ondisk = U16_MAX;
        /* We might get called multiple times on read retry: */
        b->written = 0;

        iter = mempool_alloc(&c->fill_iter, GFP_NOFS);
        sort_iter_init(iter, b, (btree_blocks(c) + 1) * 2);

        if (bch2_meta_read_fault("btree"))
                btree_err(-BCH_ERR_btree_node_read_err_must_retry,
                          c, ca, b, NULL, NULL,
                          btree_node_fault_injected,
                          "dynamic fault");

        btree_err_on(le64_to_cpu(b->data->magic) != bset_magic(c),
                     -BCH_ERR_btree_node_read_err_must_retry,
                     c, ca, b, NULL, NULL,
                     btree_node_bad_magic,
                     "bad magic: want %llx, got %llx",
                     bset_magic(c), le64_to_cpu(b->data->magic));

        if (b->key.k.type == KEY_TYPE_btree_ptr_v2) {
                struct bch_btree_ptr_v2 *bp =
                        &bkey_i_to_btree_ptr_v2(&b->key)->v;

                bch2_bpos_to_text(&buf, b->data->min_key);
                prt_str(&buf, "-");
                bch2_bpos_to_text(&buf, b->data->max_key);

                btree_err_on(b->data->keys.seq != bp->seq,
                             -BCH_ERR_btree_node_read_err_must_retry,
                             c, ca, b, NULL, NULL,
                             btree_node_bad_seq,
                             "got wrong btree node: got\n%s",
                             (printbuf_reset(&buf),
                              bch2_btree_node_header_to_text(&buf, b->data),
                              buf.buf));
        } else {
                btree_err_on(!b->data->keys.seq,
                             -BCH_ERR_btree_node_read_err_must_retry,
                             c, ca, b, NULL, NULL,
                             btree_node_bad_seq,
                             "bad btree header: seq 0\n%s",
                             (printbuf_reset(&buf),
                              bch2_btree_node_header_to_text(&buf, b->data),
                              buf.buf));
        }

        while (b->written < (ptr_written ?: btree_sectors(c))) {
                unsigned sectors;
                bool first = !b->written;

                if (first) {
                        bne = NULL;
                        i = &b->data->keys;
                } else {
                        bne = write_block(b);
                        i = &bne->keys;

                        if (i->seq != b->data->keys.seq)
                                break;
                }

                struct nonce nonce = btree_nonce(i, b->written << 9);
                bool good_csum_type = bch2_checksum_type_valid(c, BSET_CSUM_TYPE(i));

                btree_err_on(!good_csum_type,
                             bch2_csum_type_is_encryption(BSET_CSUM_TYPE(i))
                             ? -BCH_ERR_btree_node_read_err_must_retry
                             : -BCH_ERR_btree_node_read_err_want_retry,
                             c, ca, b, i, NULL,
                             bset_unknown_csum,
                             "unknown checksum type %llu", BSET_CSUM_TYPE(i));

                if (first) {
                        sectors = vstruct_sectors(b->data, c->block_bits);
                        if (btree_err_on(b->written + sectors > (ptr_written ?: btree_sectors(c)),
                                         -BCH_ERR_btree_node_read_err_fixable,
                                         c, ca, b, i, NULL,
                                         bset_past_end_of_btree_node,
                                         "bset past end of btree node (offset %u len %u but written %zu)",
                                         b->written, sectors, ptr_written ?: btree_sectors(c)))
                                i->u64s = 0;
                        if (good_csum_type) {
                                struct bch_csum csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, b->data);
                                bool csum_bad = bch2_crc_cmp(b->data->csum, csum);
                                if (csum_bad)
                                        bch2_io_error(ca, BCH_MEMBER_ERROR_checksum);

                                btree_err_on(csum_bad,
                                             -BCH_ERR_btree_node_read_err_want_retry,
                                             c, ca, b, i, NULL,
                                             bset_bad_csum,
                                             "%s",
                                             (printbuf_reset(&buf),
                                              bch2_csum_err_msg(&buf, BSET_CSUM_TYPE(i), b->data->csum, csum),
                                              buf.buf));

                                ret = bset_encrypt(c, i, b->written << 9);
                                if (bch2_fs_fatal_err_on(ret, c,
                                                         "decrypting btree node: %s", bch2_err_str(ret)))
                                        goto fsck_err;
                        }

                        btree_err_on(btree_node_type_is_extents(btree_node_type(b)) &&
                                     !BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data),
                                     -BCH_ERR_btree_node_read_err_incompatible,
                                     c, NULL, b, NULL, NULL,
                                     btree_node_unsupported_version,
                                     "btree node does not have NEW_EXTENT_OVERWRITE set");
                } else {
                        sectors = vstruct_sectors(bne, c->block_bits);
                        if (btree_err_on(b->written + sectors > (ptr_written ?: btree_sectors(c)),
                                         -BCH_ERR_btree_node_read_err_fixable,
                                         c, ca, b, i, NULL,
                                         bset_past_end_of_btree_node,
                                         "bset past end of btree node (offset %u len %u but written %zu)",
                                         b->written, sectors, ptr_written ?: btree_sectors(c)))
                                i->u64s = 0;
                        if (good_csum_type) {
                                struct bch_csum csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne);
                                bool csum_bad = bch2_crc_cmp(bne->csum, csum);
                                if (ca && csum_bad)
                                        bch2_io_error(ca, BCH_MEMBER_ERROR_checksum);

                                btree_err_on(csum_bad,
                                             -BCH_ERR_btree_node_read_err_want_retry,
                                             c, ca, b, i, NULL,
                                             bset_bad_csum,
                                             "%s",
                                             (printbuf_reset(&buf),
                                              bch2_csum_err_msg(&buf, BSET_CSUM_TYPE(i), bne->csum, csum),
                                              buf.buf));

                                ret = bset_encrypt(c, i, b->written << 9);
                                if (bch2_fs_fatal_err_on(ret, c,
                                                "decrypting btree node: %s", bch2_err_str(ret)))
                                        goto fsck_err;
                        }
                }

                b->version_ondisk = min(b->version_ondisk,
                                        le16_to_cpu(i->version));

                ret = validate_bset(c, ca, b, i, b->written, READ, failed, err_msg);
                if (ret)
                        goto fsck_err;

                if (!b->written)
                        btree_node_set_format(b, b->data->format);

                ret = validate_bset_keys(c, b, i, READ, failed, err_msg);
                if (ret)
                        goto fsck_err;

                SET_BSET_BIG_ENDIAN(i, CPU_BIG_ENDIAN);

                blacklisted = bch2_journal_seq_is_blacklisted(c,
                                        le64_to_cpu(i->journal_seq),
                                        true);

                btree_err_on(blacklisted && first,
                             -BCH_ERR_btree_node_read_err_fixable,
                             c, ca, b, i, NULL,
                             bset_blacklisted_journal_seq,
                             "first btree node bset has blacklisted journal seq (%llu)",
                             le64_to_cpu(i->journal_seq));

                btree_err_on(blacklisted && ptr_written,
                             -BCH_ERR_btree_node_read_err_fixable,
                             c, ca, b, i, NULL,
                             first_bset_blacklisted_journal_seq,
                             "found blacklisted bset (journal seq %llu) in btree node at offset %u-%u/%u",
                             le64_to_cpu(i->journal_seq),
                             b->written, b->written + sectors, ptr_written);

                b->written = min(b->written + sectors, btree_sectors(c));

                if (blacklisted && !first)
                        continue;

                sort_iter_add(iter,
                              vstruct_idx(i, 0),
                              vstruct_last(i));

                max_journal_seq = max(max_journal_seq, le64_to_cpu(i->journal_seq));
        }

        if (ptr_written) {
                btree_err_on(b->written < ptr_written,
                             -BCH_ERR_btree_node_read_err_want_retry,
                             c, ca, b, NULL, NULL,
                             btree_node_data_missing,
                             "btree node data missing: expected %u sectors, found %u",
                             ptr_written, b->written);
        } else {
                for (bne = write_block(b);
                     bset_byte_offset(b, bne) < btree_buf_bytes(b);
                     bne = (void *) bne + block_bytes(c))
                        btree_err_on(bne->keys.seq == b->data->keys.seq &&
                                     !bch2_journal_seq_is_blacklisted(c,
                                                                      le64_to_cpu(bne->keys.journal_seq),
                                                                      true),
                                     -BCH_ERR_btree_node_read_err_want_retry,
                                     c, ca, b, NULL, NULL,
                                     btree_node_bset_after_end,
                                     "found bset signature after last bset");
        }

        sorted = btree_bounce_alloc(c, btree_buf_bytes(b), &used_mempool);
        sorted->keys.u64s = 0;

        b->nr = bch2_key_sort_fix_overlapping(c, &sorted->keys, iter);
        memset((uint8_t *)(sorted + 1) + b->nr.live_u64s * sizeof(u64), 0,
                        btree_buf_bytes(b) -
                        sizeof(struct btree_node) -
                        b->nr.live_u64s * sizeof(u64));

        b->data->keys.u64s = sorted->keys.u64s;
        *sorted = *b->data;
        swap(sorted, b->data);
        set_btree_bset(b, b->set, &b->data->keys);
        b->nsets = 1;
        b->data->keys.journal_seq = cpu_to_le64(max_journal_seq);

        BUG_ON(b->nr.live_u64s != le16_to_cpu(b->data->keys.u64s));

        btree_bounce_free(c, btree_buf_bytes(b), used_mempool, sorted);

        if (updated_range)
                bch2_btree_node_drop_keys_outside_node(b);

        i = &b->data->keys;
        for (k = i->start; k != vstruct_last(i);) {
                struct bkey tmp;
                struct bkey_s u = __bkey_disassemble(b, k, &tmp);

                ret = btree_node_bkey_val_validate(c, b, u.s_c, READ);
                if (ret == -BCH_ERR_fsck_delete_bkey ||
                    (static_branch_unlikely(&bch2_inject_invalid_keys) &&
                     !bversion_cmp(u.k->bversion, MAX_VERSION))) {
                        btree_keys_account_key_drop(&b->nr, 0, k);

                        i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
                        memmove_u64s_down(k, bkey_p_next(k),
                                          (u64 *) vstruct_end(i) - (u64 *) k);
                        set_btree_bset_end(b, b->set);
                        set_btree_node_need_rewrite(b);
                        set_btree_node_need_rewrite_error(b);
                        continue;
                }
                if (ret)
                        goto fsck_err;

                if (u.k->type == KEY_TYPE_btree_ptr_v2) {
                        struct bkey_s_btree_ptr_v2 bp = bkey_s_to_btree_ptr_v2(u);

                        bp.v->mem_ptr = 0;
                }

                k = bkey_p_next(k);
        }

        bch2_bset_build_aux_tree(b, b->set, false);

        set_needs_whiteout(btree_bset_first(b), true);

        btree_node_reset_sib_u64s(b);

        scoped_guard(rcu)
                bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&b->key)), ptr) {
                        struct bch_dev *ca2 = bch2_dev_rcu(c, ptr->dev);

                        if (!ca2 || ca2->mi.state != BCH_MEMBER_STATE_rw) {
                                set_btree_node_need_rewrite(b);
                                set_btree_node_need_rewrite_degraded(b);
                        }
                }

        if (!ptr_written) {
                set_btree_node_need_rewrite(b);
                set_btree_node_need_rewrite_ptr_written_zero(b);
        }
fsck_err:
        mempool_free(iter, &c->fill_iter);
        printbuf_exit(&buf);
        bch2_time_stats_update(&c->times[BCH_TIME_btree_node_read_done], start_time);
        return ret;
}

static void btree_node_read_work(struct work_struct *work)
{
        struct btree_read_bio *rb =
                container_of(work, struct btree_read_bio, work);
        struct bch_fs *c        = rb->c;
        struct bch_dev *ca        = rb->have_ioref ? bch2_dev_have_ref(c, rb->pick.ptr.dev) : NULL;
        struct btree *b                = rb->b;
        struct bio *bio                = &rb->bio;
        struct bch_io_failures failed = { .nr = 0 };
        int ret = 0;

        struct printbuf buf = PRINTBUF;
        bch2_log_msg_start(c, &buf);

        prt_printf(&buf, "btree node read error at btree ");
        bch2_btree_pos_to_text(&buf, c, b);
        prt_newline(&buf);

        goto start;
        while (1) {
                ret = bch2_bkey_pick_read_device(c,
                                        bkey_i_to_s_c(&b->key),
                                        &failed, &rb->pick, -1);
                if (ret) {
                        set_btree_node_read_error(b);
                        break;
                }

                ca = bch2_dev_get_ioref(c, rb->pick.ptr.dev, READ, BCH_DEV_READ_REF_btree_node_read);
                rb->have_ioref                = ca != NULL;
                rb->start_time                = local_clock();
                bio_reset(bio, NULL, REQ_OP_READ|REQ_SYNC|REQ_META);
                bio->bi_iter.bi_sector        = rb->pick.ptr.offset;
                bio->bi_iter.bi_size        = btree_buf_bytes(b);

                if (rb->have_ioref) {
                        bio_set_dev(bio, ca->disk_sb.bdev);
                        submit_bio_wait(bio);
                } else {
                        bio->bi_status = BLK_STS_REMOVED;
                }

                bch2_account_io_completion(ca, BCH_MEMBER_ERROR_read,
                                           rb->start_time, !bio->bi_status);
start:
                if (rb->have_ioref)
                        enumerated_ref_put(&ca->io_ref[READ], BCH_DEV_READ_REF_btree_node_read);
                rb->have_ioref = false;

                if (bio->bi_status) {
                        bch2_mark_io_failure(&failed, &rb->pick, false);
                        continue;
                }

                ret = bch2_btree_node_read_done(c, ca, b, &failed, &buf);
                if (ret == -BCH_ERR_btree_node_read_err_want_retry ||
                    ret == -BCH_ERR_btree_node_read_err_must_retry)
                        continue;

                if (ret)
                        set_btree_node_read_error(b);

                break;
        }

        bch2_io_failures_to_text(&buf, c, &failed);

        if (btree_node_read_error(b))
                bch2_btree_lost_data(c, &buf, b->c.btree_id);

        /*
         * only print retry success if we read from a replica with no errors
         */
        if (btree_node_read_error(b))
                prt_printf(&buf, "ret %s", bch2_err_str(ret));
        else if (failed.nr) {
                if (!bch2_dev_io_failures(&failed, rb->pick.ptr.dev))
                        prt_printf(&buf, "retry success");
                else
                        prt_printf(&buf, "repair success");
        }

        if ((failed.nr ||
             btree_node_need_rewrite(b)) &&
            !btree_node_read_error(b) &&
            c->recovery.curr_pass != BCH_RECOVERY_PASS_scan_for_btree_nodes) {
                prt_printf(&buf, " (rewriting node)");
                bch2_btree_node_rewrite_async(c, b);
        }
        prt_newline(&buf);

        if (failed.nr)
                bch2_print_str_ratelimited(c, KERN_ERR, buf.buf);

        async_object_list_del(c, btree_read_bio, rb->list_idx);
        bch2_time_stats_update(&c->times[BCH_TIME_btree_node_read],
                               rb->start_time);
        bio_put(&rb->bio);
        printbuf_exit(&buf);
        clear_btree_node_read_in_flight(b);
        smp_mb__after_atomic();
        wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
}

static void btree_node_read_endio(struct bio *bio)
{
        struct btree_read_bio *rb =
                container_of(bio, struct btree_read_bio, bio);
        struct bch_fs *c        = rb->c;
        struct bch_dev *ca        = rb->have_ioref
                ? bch2_dev_have_ref(c, rb->pick.ptr.dev) : NULL;

        bch2_account_io_completion(ca, BCH_MEMBER_ERROR_read,
                                   rb->start_time, !bio->bi_status);

        queue_work(c->btree_read_complete_wq, &rb->work);
}

void bch2_btree_read_bio_to_text(struct printbuf *out, struct btree_read_bio *rbio)
{
        bch2_bio_to_text(out, &rbio->bio);
}

struct btree_node_read_all {
        struct closure                cl;
        struct bch_fs                *c;
        struct btree                *b;
        unsigned                nr;
        void                        *buf[BCH_REPLICAS_MAX];
        struct bio                *bio[BCH_REPLICAS_MAX];
        blk_status_t                err[BCH_REPLICAS_MAX];
};

static unsigned btree_node_sectors_written(struct bch_fs *c, void *data)
{
        struct btree_node *bn = data;
        struct btree_node_entry *bne;
        unsigned offset = 0;

        if (le64_to_cpu(bn->magic) !=  bset_magic(c))
                return 0;

        while (offset < btree_sectors(c)) {
                if (!offset) {
                        offset += vstruct_sectors(bn, c->block_bits);
                } else {
                        bne = data + (offset << 9);
                        if (bne->keys.seq != bn->keys.seq)
                                break;
                        offset += vstruct_sectors(bne, c->block_bits);
                }
        }

        return offset;
}

static bool btree_node_has_extra_bsets(struct bch_fs *c, unsigned offset, void *data)
{
        struct btree_node *bn = data;
        struct btree_node_entry *bne;

        if (!offset)
                return false;

        while (offset < btree_sectors(c)) {
                bne = data + (offset << 9);
                if (bne->keys.seq == bn->keys.seq)
                        return true;
                offset++;
        }

        return false;
        return offset;
}

static CLOSURE_CALLBACK(btree_node_read_all_replicas_done)
{
        closure_type(ra, struct btree_node_read_all, cl);
        struct bch_fs *c = ra->c;
        struct btree *b = ra->b;
        struct printbuf buf = PRINTBUF;
        bool dump_bset_maps = false;
        int ret = 0, best = -1, write = READ;
        unsigned i, written = 0, written2 = 0;
        __le64 seq = b->key.k.type == KEY_TYPE_btree_ptr_v2
                ? bkey_i_to_btree_ptr_v2(&b->key)->v.seq : 0;
        bool _saw_error = false, *saw_error = &_saw_error;
        struct printbuf *err_msg = NULL;
        struct bch_io_failures *failed = NULL;

        for (i = 0; i < ra->nr; i++) {
                struct btree_node *bn = ra->buf[i];

                if (ra->err[i])
                        continue;

                if (le64_to_cpu(bn->magic) != bset_magic(c) ||
                    (seq && seq != bn->keys.seq))
                        continue;

                if (best < 0) {
                        best = i;
                        written = btree_node_sectors_written(c, bn);
                        continue;
                }

                written2 = btree_node_sectors_written(c, ra->buf[i]);
                if (btree_err_on(written2 != written, -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, NULL, NULL,
                                 btree_node_replicas_sectors_written_mismatch,
                                 "btree node sectors written mismatch: %u != %u",
                                 written, written2) ||
                    btree_err_on(btree_node_has_extra_bsets(c, written2, ra->buf[i]),
                                 -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, NULL, NULL,
                                 btree_node_bset_after_end,
                                 "found bset signature after last bset") ||
                    btree_err_on(memcmp(ra->buf[best], ra->buf[i], written << 9),
                                 -BCH_ERR_btree_node_read_err_fixable,
                                 c, NULL, b, NULL, NULL,
                                 btree_node_replicas_data_mismatch,
                                 "btree node replicas content mismatch"))
                        dump_bset_maps = true;

                if (written2 > written) {
                        written = written2;
                        best = i;
                }
        }
fsck_err:
        if (dump_bset_maps) {
                for (i = 0; i < ra->nr; i++) {
                        struct btree_node *bn = ra->buf[i];
                        struct btree_node_entry *bne = NULL;
                        unsigned offset = 0, sectors;
                        bool gap = false;

                        if (ra->err[i])
                                continue;

                        printbuf_reset(&buf);

                        while (offset < btree_sectors(c)) {
                                if (!offset) {
                                        sectors = vstruct_sectors(bn, c->block_bits);
                                } else {
                                        bne = ra->buf[i] + (offset << 9);
                                        if (bne->keys.seq != bn->keys.seq)
                                                break;
                                        sectors = vstruct_sectors(bne, c->block_bits);
                                }

                                prt_printf(&buf, " %u-%u", offset, offset + sectors);
                                if (bne && bch2_journal_seq_is_blacklisted(c,
                                                        le64_to_cpu(bne->keys.journal_seq), false))
                                        prt_printf(&buf, "*");
                                offset += sectors;
                        }

                        while (offset < btree_sectors(c)) {
                                bne = ra->buf[i] + (offset << 9);
                                if (bne->keys.seq == bn->keys.seq) {
                                        if (!gap)
                                                prt_printf(&buf, " GAP");
                                        gap = true;

                                        sectors = vstruct_sectors(bne, c->block_bits);
                                        prt_printf(&buf, " %u-%u", offset, offset + sectors);
                                        if (bch2_journal_seq_is_blacklisted(c,
                                                        le64_to_cpu(bne->keys.journal_seq), false))
                                                prt_printf(&buf, "*");
                                }
                                offset++;
                        }

                        bch_err(c, "replica %u:%s", i, buf.buf);
                }
        }

        if (best >= 0) {
                memcpy(b->data, ra->buf[best], btree_buf_bytes(b));
                ret = bch2_btree_node_read_done(c, NULL, b, NULL, NULL);
        } else {
                ret = -1;
        }

        if (ret) {
                set_btree_node_read_error(b);

                struct printbuf buf = PRINTBUF;
                bch2_btree_lost_data(c, &buf, b->c.btree_id);
                if (buf.pos)
                        bch_err(c, "%s", buf.buf);
                printbuf_exit(&buf);
        } else if (*saw_error)
                bch2_btree_node_rewrite_async(c, b);

        for (i = 0; i < ra->nr; i++) {
                mempool_free(ra->buf[i], &c->btree_bounce_pool);
                bio_put(ra->bio[i]);
        }

        closure_debug_destroy(&ra->cl);
        kfree(ra);
        printbuf_exit(&buf);

        clear_btree_node_read_in_flight(b);
        smp_mb__after_atomic();
        wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
}

static void btree_node_read_all_replicas_endio(struct bio *bio)
{
        struct btree_read_bio *rb =
                container_of(bio, struct btree_read_bio, bio);
        struct bch_fs *c        = rb->c;
        struct btree_node_read_all *ra = rb->ra;

        if (rb->have_ioref) {
                struct bch_dev *ca = bch2_dev_have_ref(c, rb->pick.ptr.dev);

                bch2_latency_acct(ca, rb->start_time, READ);
                enumerated_ref_put(&ca->io_ref[READ],
                        BCH_DEV_READ_REF_btree_node_read_all_replicas);
        }

        ra->err[rb->idx] = bio->bi_status;
        closure_put(&ra->cl);
}

/*
 * XXX This allocates multiple times from the same mempools, and can deadlock
 * under sufficient memory pressure (but is only a debug path)
 */
static int btree_node_read_all_replicas(struct bch_fs *c, struct btree *b, bool sync)
{
        struct bkey_s_c k = bkey_i_to_s_c(&b->key);
        struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
        const union bch_extent_entry *entry;
        struct extent_ptr_decoded pick;
        struct btree_node_read_all *ra;
        unsigned i;

        ra = kzalloc(sizeof(*ra), GFP_NOFS);
        if (!ra)
                return bch_err_throw(c, ENOMEM_btree_node_read_all_replicas);

        closure_init(&ra->cl, NULL);
        ra->c        = c;
        ra->b        = b;
        ra->nr        = bch2_bkey_nr_ptrs(k);

        for (i = 0; i < ra->nr; i++) {
                ra->buf[i] = mempool_alloc(&c->btree_bounce_pool, GFP_NOFS);
                ra->bio[i] = bio_alloc_bioset(NULL,
                                              buf_pages(ra->buf[i], btree_buf_bytes(b)),
                                              REQ_OP_READ|REQ_SYNC|REQ_META,
                                              GFP_NOFS,
                                              &c->btree_bio);
        }

        i = 0;
        bkey_for_each_ptr_decode(k.k, ptrs, pick, entry) {
                struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ,
                                        BCH_DEV_READ_REF_btree_node_read_all_replicas);
                struct btree_read_bio *rb =
                        container_of(ra->bio[i], struct btree_read_bio, bio);
                rb->c                        = c;
                rb->b                        = b;
                rb->ra                        = ra;
                rb->start_time                = local_clock();
                rb->have_ioref                = ca != NULL;
                rb->idx                        = i;
                rb->pick                = pick;
                rb->bio.bi_iter.bi_sector = pick.ptr.offset;
                rb->bio.bi_end_io        = btree_node_read_all_replicas_endio;
                bch2_bio_map(&rb->bio, ra->buf[i], btree_buf_bytes(b));

                if (rb->have_ioref) {
                        this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_btree],
                                     bio_sectors(&rb->bio));
                        bio_set_dev(&rb->bio, ca->disk_sb.bdev);

                        closure_get(&ra->cl);
                        submit_bio(&rb->bio);
                } else {
                        ra->err[i] = BLK_STS_REMOVED;
                }

                i++;
        }

        if (sync) {
                closure_sync(&ra->cl);
                btree_node_read_all_replicas_done(&ra->cl.work);
        } else {
                continue_at(&ra->cl, btree_node_read_all_replicas_done,
                            c->btree_read_complete_wq);
        }

        return 0;
}

void bch2_btree_node_read(struct btree_trans *trans, struct btree *b,
                          bool sync)
{
        struct bch_fs *c = trans->c;
        struct extent_ptr_decoded pick;
        struct btree_read_bio *rb;
        struct bch_dev *ca;
        struct bio *bio;
        int ret;

        trace_and_count(c, btree_node_read, trans, b);

        if (static_branch_unlikely(&bch2_verify_all_btree_replicas) &&
            !btree_node_read_all_replicas(c, b, sync))
                return;

        ret = bch2_bkey_pick_read_device(c, bkey_i_to_s_c(&b->key),
                                         NULL, &pick, -1);

        if (ret <= 0) {
                bool ratelimit = true;
                struct printbuf buf = PRINTBUF;
                bch2_log_msg_start(c, &buf);

                prt_str(&buf, "btree node read error: no device to read from\n at ");
                bch2_btree_pos_to_text(&buf, c, b);
                prt_newline(&buf);
                bch2_btree_lost_data(c, &buf, b->c.btree_id);

                if (c->recovery.passes_complete & BIT_ULL(BCH_RECOVERY_PASS_check_topology) &&
                    bch2_fs_emergency_read_only2(c, &buf))
                        ratelimit = false;

                static DEFINE_RATELIMIT_STATE(rs,
                                              DEFAULT_RATELIMIT_INTERVAL,
                                              DEFAULT_RATELIMIT_BURST);
                if (!ratelimit || __ratelimit(&rs))
                        bch2_print_str(c, KERN_ERR, buf.buf);
                printbuf_exit(&buf);

                set_btree_node_read_error(b);
                clear_btree_node_read_in_flight(b);
                smp_mb__after_atomic();
                wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
                return;
        }

        ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ, BCH_DEV_READ_REF_btree_node_read);

        bio = bio_alloc_bioset(NULL,
                               buf_pages(b->data, btree_buf_bytes(b)),
                               REQ_OP_READ|REQ_SYNC|REQ_META,
                               GFP_NOFS,
                               &c->btree_bio);
        rb = container_of(bio, struct btree_read_bio, bio);
        rb->c                        = c;
        rb->b                        = b;
        rb->ra                        = NULL;
        rb->start_time                = local_clock();
        rb->have_ioref                = ca != NULL;
        rb->pick                = pick;
        INIT_WORK(&rb->work, btree_node_read_work);
        bio->bi_iter.bi_sector        = pick.ptr.offset;
        bio->bi_end_io                = btree_node_read_endio;
        bch2_bio_map(bio, b->data, btree_buf_bytes(b));

        async_object_list_add(c, btree_read_bio, rb, &rb->list_idx);

        if (rb->have_ioref) {
                this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_btree],
                             bio_sectors(bio));
                bio_set_dev(bio, ca->disk_sb.bdev);

                if (sync) {
                        submit_bio_wait(bio);
                        bch2_latency_acct(ca, rb->start_time, READ);
                        btree_node_read_work(&rb->work);
                } else {
                        submit_bio(bio);
                }
        } else {
                bio->bi_status = BLK_STS_REMOVED;

                if (sync)
                        btree_node_read_work(&rb->work);
                else
                        queue_work(c->btree_read_complete_wq, &rb->work);
        }
}

static int __bch2_btree_root_read(struct btree_trans *trans, enum btree_id id,
                                  const struct bkey_i *k, unsigned level)
{
        struct bch_fs *c = trans->c;
        struct closure cl;
        struct btree *b;
        int ret;

        closure_init_stack(&cl);

        do {
                ret = bch2_btree_cache_cannibalize_lock(trans, &cl);
                closure_sync(&cl);
        } while (ret);

        b = bch2_btree_node_mem_alloc(trans, level != 0);
        bch2_btree_cache_cannibalize_unlock(trans);

        BUG_ON(IS_ERR(b));

        bkey_copy(&b->key, k);
        BUG_ON(bch2_btree_node_hash_insert(&c->btree_cache, b, level, id));

        set_btree_node_read_in_flight(b);

        /* we can't pass the trans to read_done() for fsck errors, so it must be unlocked */
        bch2_trans_unlock(trans);
        bch2_btree_node_read(trans, b, true);

        if (btree_node_read_error(b)) {
                mutex_lock(&c->btree_cache.lock);
                bch2_btree_node_hash_remove(&c->btree_cache, b);
                mutex_unlock(&c->btree_cache.lock);

                ret = bch_err_throw(c, btree_node_read_error);
                goto err;
        }

        bch2_btree_set_root_for_read(c, b);
err:
        six_unlock_write(&b->c.lock);
        six_unlock_intent(&b->c.lock);

        return ret;
}

int bch2_btree_root_read(struct bch_fs *c, enum btree_id id,
                        const struct bkey_i *k, unsigned level)
{
        return bch2_trans_run(c, __bch2_btree_root_read(trans, id, k, level));
}

struct btree_node_scrub {
        struct bch_fs                *c;
        struct bch_dev                *ca;
        void                        *buf;
        bool                        used_mempool;
        unsigned                written;

        enum btree_id                btree;
        unsigned                level;
        struct bkey_buf                key;
        __le64                        seq;

        struct work_struct        work;
        struct bio                bio;
};

static bool btree_node_scrub_check(struct bch_fs *c, struct btree_node *data, unsigned ptr_written,
                                   struct printbuf *err)
{
        unsigned written = 0;

        if (le64_to_cpu(data->magic) != bset_magic(c)) {
                prt_printf(err, "bad magic: want %llx, got %llx",
                           bset_magic(c), le64_to_cpu(data->magic));
                return false;
        }

        while (written < (ptr_written ?: btree_sectors(c))) {
                struct btree_node_entry *bne;
                struct bset *i;
                bool first = !written;

                if (first) {
                        bne = NULL;
                        i = &data->keys;
                } else {
                        bne = (void *) data + (written << 9);
                        i = &bne->keys;

                        if (!ptr_written && i->seq != data->keys.seq)
                                break;
                }

                struct nonce nonce = btree_nonce(i, written << 9);
                bool good_csum_type = bch2_checksum_type_valid(c, BSET_CSUM_TYPE(i));

                if (first) {
                        if (good_csum_type) {
                                struct bch_csum csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, data);
                                if (bch2_crc_cmp(data->csum, csum)) {
                                        bch2_csum_err_msg(err, BSET_CSUM_TYPE(i), data->csum, csum);
                                        return false;
                                }
                        }

                        written += vstruct_sectors(data, c->block_bits);
                } else {
                        if (good_csum_type) {
                                struct bch_csum csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne);
                                if (bch2_crc_cmp(bne->csum, csum)) {
                                        bch2_csum_err_msg(err, BSET_CSUM_TYPE(i), bne->csum, csum);
                                        return false;
                                }
                        }

                        written += vstruct_sectors(bne, c->block_bits);
                }
        }

        return true;
}

static void btree_node_scrub_work(struct work_struct *work)
{
        struct btree_node_scrub *scrub = container_of(work, struct btree_node_scrub, work);
        struct bch_fs *c = scrub->c;
        struct printbuf err = PRINTBUF;

        __bch2_btree_pos_to_text(&err, c, scrub->btree, scrub->level,
                                 bkey_i_to_s_c(scrub->key.k));
        prt_newline(&err);

        if (!btree_node_scrub_check(c, scrub->buf, scrub->written, &err)) {
                int ret = bch2_trans_do(c,
                        bch2_btree_node_rewrite_key(trans, scrub->btree, scrub->level - 1,
                                                    scrub->key.k, 0));
                if (!bch2_err_matches(ret, ENOENT) &&
                    !bch2_err_matches(ret, EROFS))
                        bch_err_fn_ratelimited(c, ret);
        }

        printbuf_exit(&err);
        bch2_bkey_buf_exit(&scrub->key, c);;
        btree_bounce_free(c, c->opts.btree_node_size, scrub->used_mempool, scrub->buf);
        enumerated_ref_put(&scrub->ca->io_ref[READ], BCH_DEV_READ_REF_btree_node_scrub);
        kfree(scrub);
        enumerated_ref_put(&c->writes, BCH_WRITE_REF_btree_node_scrub);
}

static void btree_node_scrub_endio(struct bio *bio)
{
        struct btree_node_scrub *scrub = container_of(bio, struct btree_node_scrub, bio);

        queue_work(scrub->c->btree_read_complete_wq, &scrub->work);
}

int bch2_btree_node_scrub(struct btree_trans *trans,
                          enum btree_id btree, unsigned level,
                          struct bkey_s_c k, unsigned dev)
{
        if (k.k->type != KEY_TYPE_btree_ptr_v2)
                return 0;

        struct bch_fs *c = trans->c;

        if (!enumerated_ref_tryget(&c->writes, BCH_WRITE_REF_btree_node_scrub))
                return bch_err_throw(c, erofs_no_writes);

        struct extent_ptr_decoded pick;
        int ret = bch2_bkey_pick_read_device(c, k, NULL, &pick, dev);
        if (ret <= 0)
                goto err;

        struct bch_dev *ca = bch2_dev_get_ioref(c, pick.ptr.dev, READ,
                                                BCH_DEV_READ_REF_btree_node_scrub);
        if (!ca) {
                ret = bch_err_throw(c, device_offline);
                goto err;
        }

        bool used_mempool = false;
        void *buf = btree_bounce_alloc(c, c->opts.btree_node_size, &used_mempool);

        unsigned vecs = buf_pages(buf, c->opts.btree_node_size);

        struct btree_node_scrub *scrub =
                kzalloc(sizeof(*scrub) + sizeof(struct bio_vec) * vecs, GFP_KERNEL);
        if (!scrub) {
                ret = -ENOMEM;
                goto err_free;
        }

        scrub->c                = c;
        scrub->ca                = ca;
        scrub->buf                = buf;
        scrub->used_mempool        = used_mempool;
        scrub->written                = btree_ptr_sectors_written(k);

        scrub->btree                = btree;
        scrub->level                = level;
        bch2_bkey_buf_init(&scrub->key);
        bch2_bkey_buf_reassemble(&scrub->key, c, k);
        scrub->seq                = bkey_s_c_to_btree_ptr_v2(k).v->seq;

        INIT_WORK(&scrub->work, btree_node_scrub_work);

        bio_init(&scrub->bio, ca->disk_sb.bdev, scrub->bio.bi_inline_vecs, vecs, REQ_OP_READ);
        bch2_bio_map(&scrub->bio, scrub->buf, c->opts.btree_node_size);
        scrub->bio.bi_iter.bi_sector        = pick.ptr.offset;
        scrub->bio.bi_end_io                = btree_node_scrub_endio;
        submit_bio(&scrub->bio);
        return 0;
err_free:
        btree_bounce_free(c, c->opts.btree_node_size, used_mempool, buf);
        enumerated_ref_put(&ca->io_ref[READ], BCH_DEV_READ_REF_btree_node_scrub);
err:
        enumerated_ref_put(&c->writes, BCH_WRITE_REF_btree_node_scrub);
        return ret;
}

static void bch2_btree_complete_write(struct bch_fs *c, struct btree *b,
                                      struct btree_write *w)
{
        unsigned long old, new;

        old = READ_ONCE(b->will_make_reachable);
        do {
                new = old;
                if (!(old & 1))
                        break;

                new &= ~1UL;
        } while (!try_cmpxchg(&b->will_make_reachable, &old, new));

        if (old & 1)
                closure_put(&((struct btree_update *) new)->cl);

        bch2_journal_pin_drop(&c->journal, &w->journal);
}

static void __btree_node_write_done(struct bch_fs *c, struct btree *b, u64 start_time)
{
        struct btree_write *w = btree_prev_write(b);
        unsigned long old, new;
        unsigned type = 0;

        bch2_btree_complete_write(c, b, w);

        if (start_time)
                bch2_time_stats_update(&c->times[BCH_TIME_btree_node_write], start_time);

        old = READ_ONCE(b->flags);
        do {
                new = old;

                if ((old & (1U << BTREE_NODE_dirty)) &&
                    (old & (1U << BTREE_NODE_need_write)) &&
                    !(old & (1U << BTREE_NODE_never_write)) &&
                    !(old & (1U << BTREE_NODE_write_blocked)) &&
                    !(old & (1U << BTREE_NODE_will_make_reachable))) {
                        new &= ~(1U << BTREE_NODE_dirty);
                        new &= ~(1U << BTREE_NODE_need_write);
                        new |=  (1U << BTREE_NODE_write_in_flight);
                        new |=  (1U << BTREE_NODE_write_in_flight_inner);
                        new |=  (1U << BTREE_NODE_just_written);
                        new ^=  (1U << BTREE_NODE_write_idx);

                        type = new & BTREE_WRITE_TYPE_MASK;
                        new &= ~BTREE_WRITE_TYPE_MASK;
                } else {
                        new &= ~(1U << BTREE_NODE_write_in_flight);
                        new &= ~(1U << BTREE_NODE_write_in_flight_inner);
                }
        } while (!try_cmpxchg(&b->flags, &old, new));

        if (new & (1U << BTREE_NODE_write_in_flight))
                __bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|type);
        else {
                smp_mb__after_atomic();
                wake_up_bit(&b->flags, BTREE_NODE_write_in_flight);
        }
}

static void btree_node_write_done(struct bch_fs *c, struct btree *b, u64 start_time)
{
        struct btree_trans *trans = bch2_trans_get(c);

        btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read);

        /* we don't need transaction context anymore after we got the lock. */
        bch2_trans_put(trans);
        __btree_node_write_done(c, b, start_time);
        six_unlock_read(&b->c.lock);
}

static void btree_node_write_work(struct work_struct *work)
{
        struct btree_write_bio *wbio =
                container_of(work, struct btree_write_bio, work);
        struct bch_fs *c        = wbio->wbio.c;
        struct btree *b                = wbio->wbio.bio.bi_private;
        u64 start_time                = wbio->start_time;
        int ret = 0;

        btree_bounce_free(c,
                wbio->data_bytes,
                wbio->wbio.used_mempool,
                wbio->data);

        bch2_bkey_drop_ptrs(bkey_i_to_s(&wbio->key), ptr,
                bch2_dev_list_has_dev(wbio->wbio.failed, ptr->dev));

        if (!bch2_bkey_nr_ptrs(bkey_i_to_s_c(&wbio->key))) {
                ret = bch_err_throw(c, btree_node_write_all_failed);
                goto err;
        }

        if (wbio->wbio.first_btree_write) {
                if (wbio->wbio.failed.nr) {

                }
        } else {
                ret = bch2_trans_do(c,
                        bch2_btree_node_update_key_get_iter(trans, b, &wbio->key,
                                        BCH_WATERMARK_interior_updates|
                                        BCH_TRANS_COMMIT_journal_reclaim|
                                        BCH_TRANS_COMMIT_no_enospc|
                                        BCH_TRANS_COMMIT_no_check_rw,
                                        !wbio->wbio.failed.nr));
                if (ret)
                        goto err;
        }
out:
        async_object_list_del(c, btree_write_bio, wbio->list_idx);
        bio_put(&wbio->wbio.bio);
        btree_node_write_done(c, b, start_time);
        return;
err:
        set_btree_node_noevict(b);

        if (!bch2_err_matches(ret, EROFS)) {
                struct printbuf buf = PRINTBUF;
                prt_printf(&buf, "writing btree node: %s\n  ", bch2_err_str(ret));
                bch2_btree_pos_to_text(&buf, c, b);
                bch2_fs_fatal_error(c, "%s", buf.buf);
                printbuf_exit(&buf);
        }
        goto out;
}

static void btree_node_write_endio(struct bio *bio)
{
        struct bch_write_bio *wbio        = to_wbio(bio);
        struct bch_write_bio *parent        = wbio->split ? wbio->parent : NULL;
        struct bch_write_bio *orig        = parent ?: wbio;
        struct btree_write_bio *wb        = container_of(orig, struct btree_write_bio, wbio);
        struct bch_fs *c                = wbio->c;
        struct btree *b                        = wbio->bio.bi_private;
        struct bch_dev *ca                = wbio->have_ioref ? bch2_dev_have_ref(c, wbio->dev) : NULL;

        bch2_account_io_completion(ca, BCH_MEMBER_ERROR_write,
                                   wbio->submit_time, !bio->bi_status);

        if (ca && bio->bi_status) {
                struct printbuf buf = PRINTBUF;
                buf.atomic++;
                prt_printf(&buf, "btree write error: %s\n  ",
                           bch2_blk_status_to_str(bio->bi_status));
                bch2_btree_pos_to_text(&buf, c, b);
                bch_err_dev_ratelimited(ca, "%s", buf.buf);
                printbuf_exit(&buf);
        }

        if (bio->bi_status) {
                unsigned long flags;
                spin_lock_irqsave(&c->btree_write_error_lock, flags);
                bch2_dev_list_add_dev(&orig->failed, wbio->dev);
                spin_unlock_irqrestore(&c->btree_write_error_lock, flags);
        }

        /*
         * XXX: we should be using io_ref[WRITE], but we aren't retrying failed
         * btree writes yet (due to device removal/ro):
         */
        if (wbio->have_ioref)
                enumerated_ref_put(&ca->io_ref[READ],
                                   BCH_DEV_READ_REF_btree_node_write);

        if (parent) {
                bio_put(bio);
                bio_endio(&parent->bio);
                return;
        }

        clear_btree_node_write_in_flight_inner(b);
        smp_mb__after_atomic();
        wake_up_bit(&b->flags, BTREE_NODE_write_in_flight_inner);
        INIT_WORK(&wb->work, btree_node_write_work);
        queue_work(c->btree_write_complete_wq, &wb->work);
}

static int validate_bset_for_write(struct bch_fs *c, struct btree *b,
                                   struct bset *i)
{
        int ret = bch2_bkey_validate(c, bkey_i_to_s_c(&b->key),
                                     (struct bkey_validate_context) {
                                        .from        = BKEY_VALIDATE_btree_node,
                                        .level        = b->c.level + 1,
                                        .btree        = b->c.btree_id,
                                        .flags        = BCH_VALIDATE_write,
                                     });
        if (ret) {
                bch2_fs_inconsistent(c, "invalid btree node key before write");
                return ret;
        }

        ret = validate_bset_keys(c, b, i, WRITE, NULL, NULL) ?:
                validate_bset(c, NULL, b, i, b->written, WRITE, NULL, NULL);
        if (ret) {
                bch2_inconsistent_error(c);
                dump_stack();
        }

        return ret;
}

static void btree_write_submit(struct work_struct *work)
{
        struct btree_write_bio *wbio = container_of(work, struct btree_write_bio, work);
        BKEY_PADDED_ONSTACK(k, BKEY_BTREE_PTR_VAL_U64s_MAX) tmp;

        bkey_copy(&tmp.k, &wbio->key);

        bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&tmp.k)), ptr)
                ptr->offset += wbio->sector_offset;

        bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree,
                                  &tmp.k, false);
}

void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags)
{
        struct btree_write_bio *wbio;
        struct bset *i;
        struct btree_node *bn = NULL;
        struct btree_node_entry *bne = NULL;
        struct sort_iter_stack sort_iter;
        struct nonce nonce;
        unsigned bytes_to_write, sectors_to_write, bytes, u64s;
        u64 seq = 0;
        bool used_mempool;
        unsigned long old, new;
        bool validate_before_checksum = false;
        enum btree_write_type type = flags & BTREE_WRITE_TYPE_MASK;
        void *data;
        u64 start_time = local_clock();
        int ret;

        if (flags & BTREE_WRITE_ALREADY_STARTED)
                goto do_write;

        /*
         * We may only have a read lock on the btree node - the dirty bit is our
         * "lock" against racing with other threads that may be trying to start
         * a write, we do a write iff we clear the dirty bit. Since setting the
         * dirty bit requires a write lock, we can't race with other threads
         * redirtying it:
         */
        old = READ_ONCE(b->flags);
        do {
                new = old;

                if (!(old & (1 << BTREE_NODE_dirty)))
                        return;

                if ((flags & BTREE_WRITE_ONLY_IF_NEED) &&
                    !(old & (1 << BTREE_NODE_need_write)))
                        return;

                if (old &
                    ((1 << BTREE_NODE_never_write)|
                     (1 << BTREE_NODE_write_blocked)))
                        return;

                if (b->written &&
                    (old & (1 << BTREE_NODE_will_make_reachable)))
                        return;

                if (old & (1 << BTREE_NODE_write_in_flight))
                        return;

                if (flags & BTREE_WRITE_ONLY_IF_NEED)
                        type = new & BTREE_WRITE_TYPE_MASK;
                new &= ~BTREE_WRITE_TYPE_MASK;

                new &= ~(1 << BTREE_NODE_dirty);
                new &= ~(1 << BTREE_NODE_need_write);
                new |=  (1 << BTREE_NODE_write_in_flight);
                new |=  (1 << BTREE_NODE_write_in_flight_inner);
                new |=  (1 << BTREE_NODE_just_written);
                new ^=  (1 << BTREE_NODE_write_idx);
        } while (!try_cmpxchg_acquire(&b->flags, &old, new));

        if (new & (1U << BTREE_NODE_need_write))
                return;
do_write:
        BUG_ON((type == BTREE_WRITE_initial) != (b->written == 0));

        atomic_long_dec(&c->btree_cache.nr_dirty);

        BUG_ON(btree_node_fake(b));
        BUG_ON((b->will_make_reachable != 0) != !b->written);

        BUG_ON(b->written >= btree_sectors(c));
        BUG_ON(b->written & (block_sectors(c) - 1));
        BUG_ON(bset_written(b, btree_bset_last(b)));
        BUG_ON(le64_to_cpu(b->data->magic) != bset_magic(c));
        BUG_ON(memcmp(&b->data->format, &b->format, sizeof(b->format)));

        bch2_sort_whiteouts(c, b);

        sort_iter_stack_init(&sort_iter, b);

        bytes = !b->written
                ? sizeof(struct btree_node)
                : sizeof(struct btree_node_entry);

        bytes += b->whiteout_u64s * sizeof(u64);

        for_each_bset(b, t) {
                i = bset(b, t);

                if (bset_written(b, i))
                        continue;

                bytes += le16_to_cpu(i->u64s) * sizeof(u64);
                sort_iter_add(&sort_iter.iter,
                              btree_bkey_first(b, t),
                              btree_bkey_last(b, t));
                seq = max(seq, le64_to_cpu(i->journal_seq));
        }

        BUG_ON(b->written && !seq);

        /* bch2_varint_decode may read up to 7 bytes past the end of the buffer: */
        bytes += 8;

        /* buffer must be a multiple of the block size */
        bytes = round_up(bytes, block_bytes(c));

        data = btree_bounce_alloc(c, bytes, &used_mempool);

        if (!b->written) {
                bn = data;
                *bn = *b->data;
                i = &bn->keys;
        } else {
                bne = data;
                bne->keys = b->data->keys;
                i = &bne->keys;
        }

        i->journal_seq        = cpu_to_le64(seq);
        i->u64s                = 0;

        sort_iter_add(&sort_iter.iter,
                      unwritten_whiteouts_start(b),
                      unwritten_whiteouts_end(b));
        SET_BSET_SEPARATE_WHITEOUTS(i, false);

        u64s = bch2_sort_keys_keep_unwritten_whiteouts(i->start, &sort_iter.iter);
        le16_add_cpu(&i->u64s, u64s);

        b->whiteout_u64s = 0;

        BUG_ON(!b->written && i->u64s != b->data->keys.u64s);

        set_needs_whiteout(i, false);

        /* do we have data to write? */
        if (b->written && !i->u64s)
                goto nowrite;

        bytes_to_write = vstruct_end(i) - data;
        sectors_to_write = round_up(bytes_to_write, block_bytes(c)) >> 9;

        if (!b->written &&
            b->key.k.type == KEY_TYPE_btree_ptr_v2)
                BUG_ON(btree_ptr_sectors_written(bkey_i_to_s_c(&b->key)) != sectors_to_write);

        memset(data + bytes_to_write, 0,
               (sectors_to_write << 9) - bytes_to_write);

        BUG_ON(b->written + sectors_to_write > btree_sectors(c));
        BUG_ON(BSET_BIG_ENDIAN(i) != CPU_BIG_ENDIAN);
        BUG_ON(i->seq != b->data->keys.seq);

        i->version = cpu_to_le16(c->sb.version);
        SET_BSET_OFFSET(i, b->written);
        SET_BSET_CSUM_TYPE(i, bch2_meta_checksum_type(c));

        if (bch2_csum_type_is_encryption(BSET_CSUM_TYPE(i)))
                validate_before_checksum = true;

        /* validate_bset will be modifying: */
        if (le16_to_cpu(i->version) < bcachefs_metadata_version_current)
                validate_before_checksum = true;

        /* if we're going to be encrypting, check metadata validity first: */
        if (validate_before_checksum &&
            validate_bset_for_write(c, b, i))
                goto err;

        ret = bset_encrypt(c, i, b->written << 9);
        if (bch2_fs_fatal_err_on(ret, c,
                        "encrypting btree node: %s", bch2_err_str(ret)))
                goto err;

        nonce = btree_nonce(i, b->written << 9);

        if (bn)
                bn->csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bn);
        else
                bne->csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne);

        /* if we're not encrypting, check metadata after checksumming: */
        if (!validate_before_checksum &&
            validate_bset_for_write(c, b, i))
                goto err;

        /*
         * We handle btree write errors by immediately halting the journal -
         * after we've done that, we can't issue any subsequent btree writes
         * because they might have pointers to new nodes that failed to write.
         *
         * Furthermore, there's no point in doing any more btree writes because
         * with the journal stopped, we're never going to update the journal to
         * reflect that those writes were done and the data flushed from the
         * journal:
         *
         * Also on journal error, the pending write may have updates that were
         * never journalled (interior nodes, see btree_update_nodes_written()) -
         * it's critical that we don't do the write in that case otherwise we
         * will have updates visible that weren't in the journal:
         *
         * Make sure to update b->written so bch2_btree_init_next() doesn't
         * break:
         */
        if (bch2_journal_error(&c->journal) ||
            c->opts.nochanges)
                goto err;

        trace_and_count(c, btree_node_write, b, bytes_to_write, sectors_to_write);

        wbio = container_of(bio_alloc_bioset(NULL,
                                buf_pages(data, sectors_to_write << 9),
                                REQ_OP_WRITE|REQ_META,
                                GFP_NOFS,
                                &c->btree_bio),
                            struct btree_write_bio, wbio.bio);
        wbio_init(&wbio->wbio.bio);
        wbio->data                        = data;
        wbio->data_bytes                = bytes;
        wbio->sector_offset                = b->written;
        wbio->start_time                = start_time;
        wbio->wbio.c                        = c;
        wbio->wbio.used_mempool                = used_mempool;
        wbio->wbio.first_btree_write        = !b->written;
        wbio->wbio.bio.bi_end_io        = btree_node_write_endio;
        wbio->wbio.bio.bi_private        = b;

        bch2_bio_map(&wbio->wbio.bio, data, sectors_to_write << 9);

        bkey_copy(&wbio->key, &b->key);

        b->written += sectors_to_write;

        if (wbio->key.k.type == KEY_TYPE_btree_ptr_v2)
                bkey_i_to_btree_ptr_v2(&wbio->key)->v.sectors_written =
                        cpu_to_le16(b->written);

        atomic64_inc(&c->btree_write_stats[type].nr);
        atomic64_add(bytes_to_write, &c->btree_write_stats[type].bytes);

        async_object_list_add(c, btree_write_bio, wbio, &wbio->list_idx);

        INIT_WORK(&wbio->work, btree_write_submit);
        queue_work(c->btree_write_submit_wq, &wbio->work);
        return;
err:
        set_btree_node_noevict(b);
        b->written += sectors_to_write;
nowrite:
        btree_bounce_free(c, bytes, used_mempool, data);
        __btree_node_write_done(c, b, 0);
}

/*
 * Work that must be done with write lock held:
 */
bool bch2_btree_post_write_cleanup(struct bch_fs *c, struct btree *b)
{
        bool invalidated_iter = false;
        struct btree_node_entry *bne;

        if (!btree_node_just_written(b))
                return false;

        BUG_ON(b->whiteout_u64s);

        clear_btree_node_just_written(b);

        /*
         * Note: immediately after write, bset_written() doesn't work - the
         * amount of data we had to write after compaction might have been
         * smaller than the offset of the last bset.
         *
         * However, we know that all bsets have been written here, as long as
         * we're still holding the write lock:
         */

        /*
         * XXX: decide if we really want to unconditionally sort down to a
         * single bset:
         */
        if (b->nsets > 1) {
                btree_node_sort(c, b, 0, b->nsets);
                invalidated_iter = true;
        } else {
                invalidated_iter = bch2_drop_whiteouts(b, COMPACT_ALL);
        }

        for_each_bset(b, t)
                set_needs_whiteout(bset(b, t), true);

        bch2_btree_verify(c, b);

        /*
         * If later we don't unconditionally sort down to a single bset, we have
         * to ensure this is still true:
         */
        BUG_ON((void *) btree_bkey_last(b, bset_tree_last(b)) > write_block(b));

        bne = want_new_bset(c, b);
        if (bne)
                bch2_bset_init_next(b, bne);

        bch2_btree_build_aux_trees(b);

        return invalidated_iter;
}

/*
 * Use this one if the node is intent locked:
 */
void bch2_btree_node_write(struct bch_fs *c, struct btree *b,
                           enum six_lock_type lock_type_held,
                           unsigned flags)
{
        if (lock_type_held == SIX_LOCK_intent ||
            (lock_type_held == SIX_LOCK_read &&
             six_lock_tryupgrade(&b->c.lock))) {
                __bch2_btree_node_write(c, b, flags);

                /* don't cycle lock unnecessarily: */
                if (btree_node_just_written(b) &&
                    six_trylock_write(&b->c.lock)) {
                        bch2_btree_post_write_cleanup(c, b);
                        six_unlock_write(&b->c.lock);
                }

                if (lock_type_held == SIX_LOCK_read)
                        six_lock_downgrade(&b->c.lock);
        } else {
                __bch2_btree_node_write(c, b, flags);
                if (lock_type_held == SIX_LOCK_write &&
                    btree_node_just_written(b))
                        bch2_btree_post_write_cleanup(c, b);
        }
}

void bch2_btree_node_write_trans(struct btree_trans *trans, struct btree *b,
                                 enum six_lock_type lock_type_held,
                                 unsigned flags)
{
        struct bch_fs *c = trans->c;

        if (lock_type_held == SIX_LOCK_intent ||
            (lock_type_held == SIX_LOCK_read &&
             six_lock_tryupgrade(&b->c.lock))) {
                __bch2_btree_node_write(c, b, flags);

                /* don't cycle lock unnecessarily: */
                if (btree_node_just_written(b) &&
                    six_trylock_write(&b->c.lock)) {
                        bch2_btree_post_write_cleanup(c, b);
                        __bch2_btree_node_unlock_write(trans, b);
                }

                if (lock_type_held == SIX_LOCK_read)
                        six_lock_downgrade(&b->c.lock);
        } else {
                __bch2_btree_node_write(c, b, flags);
                if (lock_type_held == SIX_LOCK_write &&
                    btree_node_just_written(b))
                        bch2_btree_post_write_cleanup(c, b);
        }
}

static bool __bch2_btree_flush_all(struct bch_fs *c, unsigned flag)
{
        struct bucket_table *tbl;
        struct rhash_head *pos;
        struct btree *b;
        unsigned i;
        bool ret = false;
restart:
        rcu_read_lock();
        for_each_cached_btree(b, c, tbl, i, pos)
                if (test_bit(flag, &b->flags)) {
                        rcu_read_unlock();
                        wait_on_bit_io(&b->flags, flag, TASK_UNINTERRUPTIBLE);
                        ret = true;
                        goto restart;
                }
        rcu_read_unlock();

        return ret;
}

bool bch2_btree_flush_all_reads(struct bch_fs *c)
{
        return __bch2_btree_flush_all(c, BTREE_NODE_read_in_flight);
}

bool bch2_btree_flush_all_writes(struct bch_fs *c)
{
        return __bch2_btree_flush_all(c, BTREE_NODE_write_in_flight);
}

static const char * const bch2_btree_write_types[] = {
#define x(t, n) [n] = #t,
        BCH_BTREE_WRITE_TYPES()
        NULL
};

void bch2_btree_write_stats_to_text(struct printbuf *out, struct bch_fs *c)
{
        printbuf_tabstop_push(out, 20);
        printbuf_tabstop_push(out, 10);

        prt_printf(out, "\tnr\tsize\n");

        for (unsigned i = 0; i < BTREE_WRITE_TYPE_NR; i++) {
                u64 nr                = atomic64_read(&c->btree_write_stats[i].nr);
                u64 bytes        = atomic64_read(&c->btree_write_stats[i].bytes);

                prt_printf(out, "%s:\t%llu\t", bch2_btree_write_types[i], nr);
                prt_human_readable_u64(out, nr ? div64_u64(bytes, nr) : 0);
                prt_newline(out);
        }
}


































































    9 


    9 
    9 
    9 


    9 
    9 

    9 


    9 










































    9 


    9 




    9 
    9 
    9 
    9 






    9 













    9 
    9 



    9 








    9 



    9 
























































    9 










    9 
















    9 









    9 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020 Intel
 *
 * Based on drivers/base/devres.c
 */

#include <drm/drm_managed.h>

#include <linux/export.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/spinlock.h>

#include <drm/drm_device.h>
#include <drm/drm_print.h>

#include "drm_internal.h"

/**
 * DOC: managed resources
 *
 * Inspired by struct &device managed resources, but tied to the lifetime of
 * struct &drm_device, which can outlive the underlying physical device, usually
 * when userspace has some open files and other handles to resources still open.
 *
 * Release actions can be added with drmm_add_action(), memory allocations can
 * be done directly with drmm_kmalloc() and the related functions. Everything
 * will be released on the final drm_dev_put() in reverse order of how the
 * release actions have been added and memory has been allocated since driver
 * loading started with devm_drm_dev_alloc().
 *
 * Note that release actions and managed memory can also be added and removed
 * during the lifetime of the driver, all the functions are fully concurrent
 * safe. But it is recommended to use managed resources only for resources that
 * change rarely, if ever, during the lifetime of the &drm_device instance.
 */

struct drmres_node {
        struct list_head        entry;
        drmres_release_t        release;
        const char                *name;
        size_t                        size;
};

struct drmres {
        struct drmres_node                node;
        /*
         * Some archs want to perform DMA into kmalloc caches
         * and need a guaranteed alignment larger than
         * the alignment of a 64-bit integer.
         * Thus we use ARCH_DMA_MINALIGN for data[] which will force the same
         * alignment for struct drmres when allocated by kmalloc().
         */
        u8 __aligned(ARCH_DMA_MINALIGN) data[];
};

static void free_dr(struct drmres *dr)
{
        kfree_const(dr->node.name);
        kfree(dr);
}

void drm_managed_release(struct drm_device *dev)
{
        struct drmres *dr, *tmp;

        drm_dbg_drmres(dev, "drmres release begin\n");
        list_for_each_entry_safe(dr, tmp, &dev->managed.resources, node.entry) {
                drm_dbg_drmres(dev, "REL %p %s (%zu bytes)\n",
                               dr, dr->node.name, dr->node.size);

                if (dr->node.release)
                        dr->node.release(dev, dr->node.size ? *(void **)&dr->data : NULL);

                list_del(&dr->node.entry);
                free_dr(dr);
        }
        drm_dbg_drmres(dev, "drmres release end\n");
}

/*
 * Always inline so that kmalloc_track_caller tracks the actual interesting
 * caller outside of drm_managed.c.
 */
static __always_inline struct drmres * alloc_dr(drmres_release_t release,
                                                size_t size, gfp_t gfp, int nid)
{
        size_t tot_size;
        struct drmres *dr;

        /* We must catch any near-SIZE_MAX cases that could overflow. */
        if (unlikely(check_add_overflow(sizeof(*dr), size, &tot_size)))
                return NULL;

        dr = kmalloc_node_track_caller(tot_size, gfp, nid);
        if (unlikely(!dr))
                return NULL;

        memset(dr, 0, offsetof(struct drmres, data));

        INIT_LIST_HEAD(&dr->node.entry);
        dr->node.release = release;
        dr->node.size = size;

        return dr;
}

static void del_dr(struct drm_device *dev, struct drmres *dr)
{
        list_del_init(&dr->node.entry);

        drm_dbg_drmres(dev, "DEL %p %s (%lu bytes)\n",
                       dr, dr->node.name, (unsigned long) dr->node.size);
}

static void add_dr(struct drm_device *dev, struct drmres *dr)
{
        unsigned long flags;

        spin_lock_irqsave(&dev->managed.lock, flags);
        list_add(&dr->node.entry, &dev->managed.resources);
        spin_unlock_irqrestore(&dev->managed.lock, flags);

        drm_dbg_drmres(dev, "ADD %p %s (%lu bytes)\n",
                       dr, dr->node.name, (unsigned long) dr->node.size);
}

void drmm_add_final_kfree(struct drm_device *dev, void *container)
{
        WARN_ON(dev->managed.final_kfree);
        WARN_ON(dev < (struct drm_device *) container);
        WARN_ON(dev + 1 > (struct drm_device *) (container + ksize(container)));
        dev->managed.final_kfree = container;
}

int __drmm_add_action(struct drm_device *dev,
                      drmres_release_t action,
                      void *data, const char *name)
{
        struct drmres *dr;
        void **void_ptr;

        dr = alloc_dr(action, data ? sizeof(void*) : 0,
                      GFP_KERNEL | __GFP_ZERO,
                      dev_to_node(dev->dev));
        if (!dr) {
                drm_dbg_drmres(dev, "failed to add action %s for %p\n",
                               name, data);
                return -ENOMEM;
        }

        dr->node.name = kstrdup_const(name, GFP_KERNEL);
        if (data) {
                void_ptr = (void **)&dr->data;
                *void_ptr = data;
        }

        add_dr(dev, dr);

        return 0;
}
EXPORT_SYMBOL(__drmm_add_action);

int __drmm_add_action_or_reset(struct drm_device *dev,
                               drmres_release_t action,
                               void *data, const char *name)
{
        int ret;

        ret = __drmm_add_action(dev, action, data, name);
        if (ret)
                action(dev, data);

        return ret;
}
EXPORT_SYMBOL(__drmm_add_action_or_reset);

/**
 * drmm_release_action - release a managed action from a &drm_device
 * @dev: DRM device
 * @action: function which would be called when @dev is released
 * @data: opaque pointer, passed to @action
 *
 * This function calls the @action previously added by drmm_add_action()
 * immediately.
 * The @action is removed from the list of cleanup actions for @dev,
 * which means that it won't be called in the final drm_dev_put().
 */
void drmm_release_action(struct drm_device *dev,
                         drmres_release_t action,
                         void *data)
{
        struct drmres *dr_match = NULL, *dr;
        unsigned long flags;

        spin_lock_irqsave(&dev->managed.lock, flags);
        list_for_each_entry_reverse(dr, &dev->managed.resources, node.entry) {
                if (dr->node.release == action) {
                        if (!data || *(void **)dr->data == data) {
                                dr_match = dr;
                                del_dr(dev, dr_match);
                                break;
                        }
                }
        }
        spin_unlock_irqrestore(&dev->managed.lock, flags);

        if (WARN_ON(!dr_match))
                return;

        action(dev, data);

        free_dr(dr_match);
}
EXPORT_SYMBOL(drmm_release_action);

/**
 * drmm_kmalloc - &drm_device managed kmalloc()
 * @dev: DRM device
 * @size: size of the memory allocation
 * @gfp: GFP allocation flags
 *
 * This is a &drm_device managed version of kmalloc(). The allocated memory is
 * automatically freed on the final drm_dev_put(). Memory can also be freed
 * before the final drm_dev_put() by calling drmm_kfree().
 */
void *drmm_kmalloc(struct drm_device *dev, size_t size, gfp_t gfp)
{
        struct drmres *dr;

        dr = alloc_dr(NULL, size, gfp, dev_to_node(dev->dev));
        if (!dr) {
                drm_dbg_drmres(dev, "failed to allocate %zu bytes, %u flags\n",
                               size, gfp);
                return NULL;
        }
        dr->node.name = kstrdup_const("kmalloc", gfp);

        add_dr(dev, dr);

        return dr->data;
}
EXPORT_SYMBOL(drmm_kmalloc);

/**
 * drmm_kstrdup - &drm_device managed kstrdup()
 * @dev: DRM device
 * @s: 0-terminated string to be duplicated
 * @gfp: GFP allocation flags
 *
 * This is a &drm_device managed version of kstrdup(). The allocated memory is
 * automatically freed on the final drm_dev_put() and works exactly like a
 * memory allocation obtained by drmm_kmalloc().
 */
char *drmm_kstrdup(struct drm_device *dev, const char *s, gfp_t gfp)
{
        size_t size;
        char *buf;

        if (!s)
                return NULL;

        size = strlen(s) + 1;
        buf = drmm_kmalloc(dev, size, gfp);
        if (buf)
                memcpy(buf, s, size);
        return buf;
}
EXPORT_SYMBOL_GPL(drmm_kstrdup);

/**
 * drmm_kfree - &drm_device managed kfree()
 * @dev: DRM device
 * @data: memory allocation to be freed
 *
 * This is a &drm_device managed version of kfree() which can be used to
 * release memory allocated through drmm_kmalloc() or any of its related
 * functions before the final drm_dev_put() of @dev.
 */
void drmm_kfree(struct drm_device *dev, void *data)
{
        struct drmres *dr_match = NULL, *dr;
        unsigned long flags;

        if (!data)
                return;

        spin_lock_irqsave(&dev->managed.lock, flags);
        list_for_each_entry(dr, &dev->managed.resources, node.entry) {
                if (dr->data == data) {
                        dr_match = dr;
                        del_dr(dev, dr_match);
                        break;
                }
        }
        spin_unlock_irqrestore(&dev->managed.lock, flags);

        if (WARN_ON(!dr_match))
                return;

        free_dr(dr_match);
}
EXPORT_SYMBOL(drmm_kfree);

void __drmm_mutex_release(struct drm_device *dev, void *res)
{
        struct mutex *lock = res;

        mutex_destroy(lock);
}
EXPORT_SYMBOL(__drmm_mutex_release);

void __drmm_workqueue_release(struct drm_device *device, void *res)
{
        struct workqueue_struct *wq = res;

        destroy_workqueue(wq);
}
EXPORT_SYMBOL(__drmm_workqueue_release);













































































































































   11 

   11 
























  521 









   97 











  969 
   71 



























































   29 




   20 
   16 

   16 
    8 
    8 

   48 
   56 
   56 


   56 






   49 

   20 
   16 






   17 


   19 
   15 

   10 
    4 
    6 



































































    5 






    8 
    5 
    5 


    8 


    6 

    2 
















  389 








    2 






 3507 





























































  266 



 5252 
 5266 
 5253 
 5259 








 23588 
 23577 

 23589 









 1862 
    4 

































































































































































































    2 

    2 






























    5 





 5088 
 5010 







  323 


   22 



 8985 


 8802 
  428 








































































































































































































    8 












    6 




    2 





























 1805 
   47 















 1028 
   49 























    1 



  266 



   15 


















  237 




   60 






   59 


   21 







    1 

  276 
  276 

























































































































































































































































































































































  231 





  145 





   91 





   85 



    1 






    4 










   10 



















































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
/* SPDX-License-Identifier: GPL-2.0-or-later */
/* internal.h: mm/ internal definitions
 *
 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */
#ifndef __MM_INTERNAL_H
#define __MM_INTERNAL_H

#include <linux/fs.h>
#include <linux/khugepaged.h>
#include <linux/mm.h>
#include <linux/mm_inline.h>
#include <linux/pagemap.h>
#include <linux/pagewalk.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/swap_cgroup.h>
#include <linux/tracepoint-defs.h>

/* Internal core VMA manipulation functions. */
#include "vma.h"

struct folio_batch;

/*
 * Maintains state across a page table move. The operation assumes both source
 * and destination VMAs already exist and are specified by the user.
 *
 * Partial moves are permitted, but the old and new ranges must both reside
 * within a VMA.
 *
 * mmap lock must be held in write and VMA write locks must be held on any VMA
 * that is visible.
 *
 * Use the PAGETABLE_MOVE() macro to initialise this struct.
 *
 * The old_addr and new_addr fields are updated as the page table move is
 * executed.
 *
 * NOTE: The page table move is affected by reading from [old_addr, old_end),
 * and old_addr may be updated for better page table alignment, so len_in
 * represents the length of the range being copied as specified by the user.
 */
struct pagetable_move_control {
        struct vm_area_struct *old; /* Source VMA. */
        struct vm_area_struct *new; /* Destination VMA. */
        struct vm_area_struct *relocate_locked; /* VMA which is rmap locked. */
        unsigned long old_addr; /* Address from which the move begins. */
        unsigned long old_end; /* Exclusive address at which old range ends. */
        unsigned long new_addr; /* Address to move page tables to. */
        unsigned long len_in; /* Bytes to remap specified by user. */

        bool need_rmap_locks; /* Do rmap locks need to be taken? */
        bool for_stack; /* Is this an early temp stack being moved? */
};

#define PAGETABLE_MOVE(name, old_, new_, old_addr_, new_addr_, len_)        \
        struct pagetable_move_control name = {                                \
                .old = old_,                                                \
                .new = new_,                                                \
                .old_addr = old_addr_,                                        \
                .old_end = (old_addr_) + (len_),                        \
                .new_addr = new_addr_,                                        \
                .len_in = len_,                                                \
        }

/*
 * The set of flags that only affect watermark checking and reclaim
 * behaviour. This is used by the MM to obey the caller constraints
 * about IO, FS and watermark checking while ignoring placement
 * hints such as HIGHMEM usage.
 */
#define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\
                        __GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\
                        __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\
                        __GFP_NOLOCKDEP)

/* The GFP flags allowed during early boot */
#define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS))

/* Control allocation cpuset and node placement constraints */
#define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)

/* Do not use these with a slab allocator */
#define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)

/*
 * Different from WARN_ON_ONCE(), no warning will be issued
 * when we specify __GFP_NOWARN.
 */
#define WARN_ON_ONCE_GFP(cond, gfp)        ({                                \
        static bool __section(".data..once") __warned;                        \
        int __ret_warn_once = !!(cond);                                        \
                                                                        \
        if (unlikely(!(gfp & __GFP_NOWARN) && __ret_warn_once && !__warned)) { \
                __warned = true;                                        \
                WARN_ON(1);                                                \
        }                                                                \
        unlikely(__ret_warn_once);                                        \
})

void page_writeback_init(void);

/*
 * If a 16GB hugetlb folio were mapped by PTEs of all of its 4kB pages,
 * its nr_pages_mapped would be 0x400000: choose the ENTIRELY_MAPPED bit
 * above that range, instead of 2*(PMD_SIZE/PAGE_SIZE).  Hugetlb currently
 * leaves nr_pages_mapped at 0, but avoid surprise if it participates later.
 */
#define ENTIRELY_MAPPED                0x800000
#define FOLIO_PAGES_MAPPED        (ENTIRELY_MAPPED - 1)

/*
 * Flags passed to __show_mem() and show_free_areas() to suppress output in
 * various contexts.
 */
#define SHOW_MEM_FILTER_NODES                (0x0001u)        /* disallowed nodes */

/*
 * How many individual pages have an elevated _mapcount.  Excludes
 * the folio's entire_mapcount.
 *
 * Don't use this function outside of debugging code.
 */
static inline int folio_nr_pages_mapped(const struct folio *folio)
{
        if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT))
                return -1;
        return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED;
}

/*
 * Retrieve the first entry of a folio based on a provided entry within the
 * folio. We cannot rely on folio->swap as there is no guarantee that it has
 * been initialized. Used for calling arch_swap_restore()
 */
static inline swp_entry_t folio_swap(swp_entry_t entry,
                const struct folio *folio)
{
        swp_entry_t swap = {
                .val = ALIGN_DOWN(entry.val, folio_nr_pages(folio)),
        };

        return swap;
}

static inline void *folio_raw_mapping(const struct folio *folio)
{
        unsigned long mapping = (unsigned long)folio->mapping;

        return (void *)(mapping & ~PAGE_MAPPING_FLAGS);
}

/*
 * This is a file-backed mapping, and is about to be memory mapped - invoke its
 * mmap hook and safely handle error conditions. On error, VMA hooks will be
 * mutated.
 *
 * @file: File which backs the mapping.
 * @vma:  VMA which we are mapping.
 *
 * Returns: 0 if success, error otherwise.
 */
static inline int mmap_file(struct file *file, struct vm_area_struct *vma)
{
        int err = vfs_mmap(file, vma);

        if (likely(!err))
                return 0;

        /*
         * OK, we tried to call the file hook for mmap(), but an error
         * arose. The mapping is in an inconsistent state and we most not invoke
         * any further hooks on it.
         */
        vma->vm_ops = &vma_dummy_vm_ops;

        return err;
}

/*
 * If the VMA has a close hook then close it, and since closing it might leave
 * it in an inconsistent state which makes the use of any hooks suspect, clear
 * them down by installing dummy empty hooks.
 */
static inline void vma_close(struct vm_area_struct *vma)
{
        if (vma->vm_ops && vma->vm_ops->close) {
                vma->vm_ops->close(vma);

                /*
                 * The mapping is in an inconsistent state, and no further hooks
                 * may be invoked upon it.
                 */
                vma->vm_ops = &vma_dummy_vm_ops;
        }
}

#ifdef CONFIG_MMU

/* Flags for folio_pte_batch(). */
typedef int __bitwise fpb_t;

/* Compare PTEs after pte_mkclean(), ignoring the dirty bit. */
#define FPB_IGNORE_DIRTY                ((__force fpb_t)BIT(0))

/* Compare PTEs after pte_clear_soft_dirty(), ignoring the soft-dirty bit. */
#define FPB_IGNORE_SOFT_DIRTY                ((__force fpb_t)BIT(1))

static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags)
{
        if (flags & FPB_IGNORE_DIRTY)
                pte = pte_mkclean(pte);
        if (likely(flags & FPB_IGNORE_SOFT_DIRTY))
                pte = pte_clear_soft_dirty(pte);
        return pte_wrprotect(pte_mkold(pte));
}

/**
 * folio_pte_batch - detect a PTE batch for a large folio
 * @folio: The large folio to detect a PTE batch for.
 * @addr: The user virtual address the first page is mapped at.
 * @start_ptep: Page table pointer for the first entry.
 * @pte: Page table entry for the first page.
 * @max_nr: The maximum number of table entries to consider.
 * @flags: Flags to modify the PTE batch semantics.
 * @any_writable: Optional pointer to indicate whether any entry except the
 *                  first one is writable.
 * @any_young: Optional pointer to indicate whether any entry except the
 *                  first one is young.
 * @any_dirty: Optional pointer to indicate whether any entry except the
 *                  first one is dirty.
 *
 * Detect a PTE batch: consecutive (present) PTEs that map consecutive
 * pages of the same large folio.
 *
 * All PTEs inside a PTE batch have the same PTE bits set, excluding the PFN,
 * the accessed bit, writable bit, dirty bit (with FPB_IGNORE_DIRTY) and
 * soft-dirty bit (with FPB_IGNORE_SOFT_DIRTY).
 *
 * start_ptep must map any page of the folio. max_nr must be at least one and
 * must be limited by the caller so scanning cannot exceed a single page table.
 *
 * Return: the number of table entries in the batch.
 */
static inline int folio_pte_batch(struct folio *folio, unsigned long addr,
                pte_t *start_ptep, pte_t pte, int max_nr, fpb_t flags,
                bool *any_writable, bool *any_young, bool *any_dirty)
{
        pte_t expected_pte, *ptep;
        bool writable, young, dirty;
        int nr, cur_nr;

        if (any_writable)
                *any_writable = false;
        if (any_young)
                *any_young = false;
        if (any_dirty)
                *any_dirty = false;

        VM_WARN_ON_FOLIO(!pte_present(pte), folio);
        VM_WARN_ON_FOLIO(!folio_test_large(folio) || max_nr < 1, folio);
        VM_WARN_ON_FOLIO(page_folio(pfn_to_page(pte_pfn(pte))) != folio, folio);

        /* Limit max_nr to the actual remaining PFNs in the folio we could batch. */
        max_nr = min_t(unsigned long, max_nr,
                       folio_pfn(folio) + folio_nr_pages(folio) - pte_pfn(pte));

        nr = pte_batch_hint(start_ptep, pte);
        expected_pte = __pte_batch_clear_ignored(pte_advance_pfn(pte, nr), flags);
        ptep = start_ptep + nr;

        while (nr < max_nr) {
                pte = ptep_get(ptep);
                if (any_writable)
                        writable = !!pte_write(pte);
                if (any_young)
                        young = !!pte_young(pte);
                if (any_dirty)
                        dirty = !!pte_dirty(pte);
                pte = __pte_batch_clear_ignored(pte, flags);

                if (!pte_same(pte, expected_pte))
                        break;

                if (any_writable)
                        *any_writable |= writable;
                if (any_young)
                        *any_young |= young;
                if (any_dirty)
                        *any_dirty |= dirty;

                cur_nr = pte_batch_hint(ptep, pte);
                expected_pte = pte_advance_pfn(expected_pte, cur_nr);
                ptep += cur_nr;
                nr += cur_nr;
        }

        return min(nr, max_nr);
}

/**
 * pte_move_swp_offset - Move the swap entry offset field of a swap pte
 *         forward or backward by delta
 * @pte: The initial pte state; is_swap_pte(pte) must be true and
 *         non_swap_entry() must be false.
 * @delta: The direction and the offset we are moving; forward if delta
 *         is positive; backward if delta is negative
 *
 * Moves the swap offset, while maintaining all other fields, including
 * swap type, and any swp pte bits. The resulting pte is returned.
 */
static inline pte_t pte_move_swp_offset(pte_t pte, long delta)
{
        swp_entry_t entry = pte_to_swp_entry(pte);
        pte_t new = __swp_entry_to_pte(__swp_entry(swp_type(entry),
                                                   (swp_offset(entry) + delta)));

        if (pte_swp_soft_dirty(pte))
                new = pte_swp_mksoft_dirty(new);
        if (pte_swp_exclusive(pte))
                new = pte_swp_mkexclusive(new);
        if (pte_swp_uffd_wp(pte))
                new = pte_swp_mkuffd_wp(new);

        return new;
}


/**
 * pte_next_swp_offset - Increment the swap entry offset field of a swap pte.
 * @pte: The initial pte state; is_swap_pte(pte) must be true and
 *         non_swap_entry() must be false.
 *
 * Increments the swap offset, while maintaining all other fields, including
 * swap type, and any swp pte bits. The resulting pte is returned.
 */
static inline pte_t pte_next_swp_offset(pte_t pte)
{
        return pte_move_swp_offset(pte, 1);
}

/**
 * swap_pte_batch - detect a PTE batch for a set of contiguous swap entries
 * @start_ptep: Page table pointer for the first entry.
 * @max_nr: The maximum number of table entries to consider.
 * @pte: Page table entry for the first entry.
 *
 * Detect a batch of contiguous swap entries: consecutive (non-present) PTEs
 * containing swap entries all with consecutive offsets and targeting the same
 * swap type, all with matching swp pte bits.
 *
 * max_nr must be at least one and must be limited by the caller so scanning
 * cannot exceed a single page table.
 *
 * Return: the number of table entries in the batch.
 */
static inline int swap_pte_batch(pte_t *start_ptep, int max_nr, pte_t pte)
{
        pte_t expected_pte = pte_next_swp_offset(pte);
        const pte_t *end_ptep = start_ptep + max_nr;
        swp_entry_t entry = pte_to_swp_entry(pte);
        pte_t *ptep = start_ptep + 1;
        unsigned short cgroup_id;

        VM_WARN_ON(max_nr < 1);
        VM_WARN_ON(!is_swap_pte(pte));
        VM_WARN_ON(non_swap_entry(entry));

        cgroup_id = lookup_swap_cgroup_id(entry);
        while (ptep < end_ptep) {
                pte = ptep_get(ptep);

                if (!pte_same(pte, expected_pte))
                        break;
                if (lookup_swap_cgroup_id(pte_to_swp_entry(pte)) != cgroup_id)
                        break;
                expected_pte = pte_next_swp_offset(expected_pte);
                ptep++;
        }

        return ptep - start_ptep;
}
#endif /* CONFIG_MMU */

void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
                                                int nr_throttled);
static inline void acct_reclaim_writeback(struct folio *folio)
{
        pg_data_t *pgdat = folio_pgdat(folio);
        int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled);

        if (nr_throttled)
                __acct_reclaim_writeback(pgdat, folio, nr_throttled);
}

static inline void wake_throttle_isolated(pg_data_t *pgdat)
{
        wait_queue_head_t *wqh;

        wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED];
        if (waitqueue_active(wqh))
                wake_up(wqh);
}

vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf);
static inline vm_fault_t vmf_anon_prepare(struct vm_fault *vmf)
{
        vm_fault_t ret = __vmf_anon_prepare(vmf);

        if (unlikely(ret & VM_FAULT_RETRY))
                vma_end_read(vmf->vma);
        return ret;
}

vm_fault_t do_swap_page(struct vm_fault *vmf);
void folio_rotate_reclaimable(struct folio *folio);
bool __folio_end_writeback(struct folio *folio);
void deactivate_file_folio(struct folio *folio);
void folio_activate(struct folio *folio);

void free_pgtables(struct mmu_gather *tlb, struct ma_state *mas,
                   struct vm_area_struct *start_vma, unsigned long floor,
                   unsigned long ceiling, bool mm_wr_locked);
void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);

struct zap_details;
void unmap_page_range(struct mmu_gather *tlb,
                             struct vm_area_struct *vma,
                             unsigned long addr, unsigned long end,
                             struct zap_details *details);
void zap_page_range_single_batched(struct mmu_gather *tlb,
                struct vm_area_struct *vma, unsigned long addr,
                unsigned long size, struct zap_details *details);
int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio,
                           gfp_t gfp);

void page_cache_ra_order(struct readahead_control *, struct file_ra_state *);
void force_page_cache_ra(struct readahead_control *, unsigned long nr);
static inline void force_page_cache_readahead(struct address_space *mapping,
                struct file *file, pgoff_t index, unsigned long nr_to_read)
{
        DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index);
        force_page_cache_ra(&ractl, nr_to_read);
}

unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
                pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
                pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices);
void filemap_free_folio(struct address_space *mapping, struct folio *folio);
int truncate_inode_folio(struct address_space *mapping, struct folio *folio);
bool truncate_inode_partial_folio(struct folio *folio, loff_t start,
                loff_t end);
long mapping_evict_folio(struct address_space *mapping, struct folio *folio);
unsigned long mapping_try_invalidate(struct address_space *mapping,
                pgoff_t start, pgoff_t end, unsigned long *nr_failed);

/**
 * folio_evictable - Test whether a folio is evictable.
 * @folio: The folio to test.
 *
 * Test whether @folio is evictable -- i.e., should be placed on
 * active/inactive lists vs unevictable list.
 *
 * Reasons folio might not be evictable:
 * 1. folio's mapping marked unevictable
 * 2. One of the pages in the folio is part of an mlocked VMA
 */
static inline bool folio_evictable(struct folio *folio)
{
        bool ret;

        /* Prevent address_space of inode and swap cache from being freed */
        rcu_read_lock();
        ret = !mapping_unevictable(folio_mapping(folio)) &&
                        !folio_test_mlocked(folio);
        rcu_read_unlock();
        return ret;
}

/*
 * Turn a non-refcounted page (->_refcount == 0) into refcounted with
 * a count of one.
 */
static inline void set_page_refcounted(struct page *page)
{
        VM_BUG_ON_PAGE(PageTail(page), page);
        VM_BUG_ON_PAGE(page_ref_count(page), page);
        set_page_count(page, 1);
}

/*
 * Return true if a folio needs ->release_folio() calling upon it.
 */
static inline bool folio_needs_release(struct folio *folio)
{
        struct address_space *mapping = folio_mapping(folio);

        return folio_has_private(folio) ||
                (mapping && mapping_release_always(mapping));
}

extern unsigned long highest_memmap_pfn;

/*
 * Maximum number of reclaim retries without progress before the OOM
 * killer is consider the only way forward.
 */
#define MAX_RECLAIM_RETRIES 16

/*
 * in mm/vmscan.c:
 */
bool folio_isolate_lru(struct folio *folio);
void folio_putback_lru(struct folio *folio);
extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);

/*
 * in mm/rmap.c:
 */
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);

/*
 * in mm/page_alloc.c
 */
#define K(x) ((x) << (PAGE_SHIFT-10))

extern char * const zone_names[MAX_NR_ZONES];

/* perform sanity checks on struct pages being allocated or freed */
DECLARE_STATIC_KEY_MAYBE(CONFIG_DEBUG_VM, check_pages_enabled);

extern int min_free_kbytes;
extern int defrag_mode;

void setup_per_zone_wmarks(void);
void calculate_min_free_kbytes(void);
int __meminit init_per_zone_wmark_min(void);
void page_alloc_sysctl_init(void);

/*
 * Structure for holding the mostly immutable allocation parameters passed
 * between functions involved in allocations, including the alloc_pages*
 * family of functions.
 *
 * nodemask, migratetype and highest_zoneidx are initialized only once in
 * __alloc_pages() and then never change.
 *
 * zonelist, preferred_zone and highest_zoneidx are set first in
 * __alloc_pages() for the fast path, and might be later changed
 * in __alloc_pages_slowpath(). All other functions pass the whole structure
 * by a const pointer.
 */
struct alloc_context {
        struct zonelist *zonelist;
        nodemask_t *nodemask;
        struct zoneref *preferred_zoneref;
        int migratetype;

        /*
         * highest_zoneidx represents highest usable zone index of
         * the allocation request. Due to the nature of the zone,
         * memory on lower zone than the highest_zoneidx will be
         * protected by lowmem_reserve[highest_zoneidx].
         *
         * highest_zoneidx is also used by reclaim/compaction to limit
         * the target zone since higher zone than this index cannot be
         * usable for this allocation request.
         */
        enum zone_type highest_zoneidx;
        bool spread_dirty_pages;
};

/*
 * This function returns the order of a free page in the buddy system. In
 * general, page_zone(page)->lock must be held by the caller to prevent the
 * page from being allocated in parallel and returning garbage as the order.
 * If a caller does not hold page_zone(page)->lock, it must guarantee that the
 * page cannot be allocated or merged in parallel. Alternatively, it must
 * handle invalid values gracefully, and use buddy_order_unsafe() below.
 */
static inline unsigned int buddy_order(struct page *page)
{
        /* PageBuddy() must be checked by the caller */
        return page_private(page);
}

/*
 * Like buddy_order(), but for callers who cannot afford to hold the zone lock.
 * PageBuddy() should be checked first by the caller to minimize race window,
 * and invalid values must be handled gracefully.
 *
 * READ_ONCE is used so that if the caller assigns the result into a local
 * variable and e.g. tests it for valid range before using, the compiler cannot
 * decide to remove the variable and inline the page_private(page) multiple
 * times, potentially observing different values in the tests and the actual
 * use of the result.
 */
#define buddy_order_unsafe(page)        READ_ONCE(page_private(page))

/*
 * This function checks whether a page is free && is the buddy
 * we can coalesce a page and its buddy if
 * (a) the buddy is not in a hole (check before calling!) &&
 * (b) the buddy is in the buddy system &&
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
 *
 * For recording whether a page is in the buddy system, we set PageBuddy.
 * Setting, clearing, and testing PageBuddy is serialized by zone->lock.
 *
 * For recording page's order, we use page_private(page).
 */
static inline bool page_is_buddy(struct page *page, struct page *buddy,
                                 unsigned int order)
{
        if (!page_is_guard(buddy) && !PageBuddy(buddy))
                return false;

        if (buddy_order(buddy) != order)
                return false;

        /*
         * zone check is done late to avoid uselessly calculating
         * zone/node ids for pages that could never merge.
         */
        if (page_zone_id(page) != page_zone_id(buddy))
                return false;

        VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

        return true;
}

/*
 * Locate the struct page for both the matching buddy in our
 * pair (buddy1) and the combined O(n+1) page they form (page).
 *
 * 1) Any buddy B1 will have an order O twin B2 which satisfies
 * the following equation:
 *     B2 = B1 ^ (1 << O)
 * For example, if the starting buddy (buddy2) is #8 its order
 * 1 buddy is #10:
 *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
 *
 * 2) Any buddy B will have an order O+1 parent P which
 * satisfies the following equation:
 *     P = B & ~(1 << O)
 *
 * Assumption: *_mem_map is contiguous at least up to MAX_PAGE_ORDER
 */
static inline unsigned long
__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
{
        return page_pfn ^ (1 << order);
}

/*
 * Find the buddy of @page and validate it.
 * @page: The input page
 * @pfn: The pfn of the page, it saves a call to page_to_pfn() when the
 *       function is used in the performance-critical __free_one_page().
 * @order: The order of the page
 * @buddy_pfn: The output pointer to the buddy pfn, it also saves a call to
 *             page_to_pfn().
 *
 * The found buddy can be a non PageBuddy, out of @page's zone, or its order is
 * not the same as @page. The validation is necessary before use it.
 *
 * Return: the found buddy page or NULL if not found.
 */
static inline struct page *find_buddy_page_pfn(struct page *page,
                        unsigned long pfn, unsigned int order, unsigned long *buddy_pfn)
{
        unsigned long __buddy_pfn = __find_buddy_pfn(pfn, order);
        struct page *buddy;

        buddy = page + (__buddy_pfn - pfn);
        if (buddy_pfn)
                *buddy_pfn = __buddy_pfn;

        if (page_is_buddy(page, buddy, order))
                return buddy;
        return NULL;
}

extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
                                unsigned long end_pfn, struct zone *zone);

static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
                                unsigned long end_pfn, struct zone *zone)
{
        if (zone->contiguous)
                return pfn_to_page(start_pfn);

        return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
}

void set_zone_contiguous(struct zone *zone);
bool pfn_range_intersects_zones(int nid, unsigned long start_pfn,
                           unsigned long nr_pages);

static inline void clear_zone_contiguous(struct zone *zone)
{
        zone->contiguous = false;
}

extern int __isolate_free_page(struct page *page, unsigned int order);
extern void __putback_isolated_page(struct page *page, unsigned int order,
                                    int mt);
extern void memblock_free_pages(struct page *page, unsigned long pfn,
                                        unsigned int order);
extern void __free_pages_core(struct page *page, unsigned int order,
                enum meminit_context context);

/*
 * This will have no effect, other than possibly generating a warning, if the
 * caller passes in a non-large folio.
 */
static inline void folio_set_order(struct folio *folio, unsigned int order)
{
        if (WARN_ON_ONCE(!order || !folio_test_large(folio)))
                return;

        folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order;
#ifdef NR_PAGES_IN_LARGE_FOLIO
        folio->_nr_pages = 1U << order;
#endif
}

bool __folio_unqueue_deferred_split(struct folio *folio);
static inline bool folio_unqueue_deferred_split(struct folio *folio)
{
        if (folio_order(folio) <= 1 || !folio_test_large_rmappable(folio))
                return false;

        /*
         * At this point, there is no one trying to add the folio to
         * deferred_list. If folio is not in deferred_list, it's safe
         * to check without acquiring the split_queue_lock.
         */
        if (data_race(list_empty(&folio->_deferred_list)))
                return false;

        return __folio_unqueue_deferred_split(folio);
}

static inline struct folio *page_rmappable_folio(struct page *page)
{
        struct folio *folio = (struct folio *)page;

        if (folio && folio_test_large(folio))
                folio_set_large_rmappable(folio);
        return folio;
}

static inline void prep_compound_head(struct page *page, unsigned int order)
{
        struct folio *folio = (struct folio *)page;

        folio_set_order(folio, order);
        atomic_set(&folio->_large_mapcount, -1);
        if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT))
                atomic_set(&folio->_nr_pages_mapped, 0);
        if (IS_ENABLED(CONFIG_MM_ID)) {
                folio->_mm_ids = 0;
                folio->_mm_id_mapcount[0] = -1;
                folio->_mm_id_mapcount[1] = -1;
        }
        if (IS_ENABLED(CONFIG_64BIT) || order > 1) {
                atomic_set(&folio->_pincount, 0);
                atomic_set(&folio->_entire_mapcount, -1);
        }
        if (order > 1)
                INIT_LIST_HEAD(&folio->_deferred_list);
}

static inline void prep_compound_tail(struct page *head, int tail_idx)
{
        struct page *p = head + tail_idx;

        p->mapping = TAIL_MAPPING;
        set_compound_head(p, head);
        set_page_private(p, 0);
}

void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags);
extern bool free_pages_prepare(struct page *page, unsigned int order);

extern int user_min_free_kbytes;

struct page *__alloc_frozen_pages_noprof(gfp_t, unsigned int order, int nid,
                nodemask_t *);
#define __alloc_frozen_pages(...) \
        alloc_hooks(__alloc_frozen_pages_noprof(__VA_ARGS__))
void free_frozen_pages(struct page *page, unsigned int order);
void free_unref_folios(struct folio_batch *fbatch);

#ifdef CONFIG_NUMA
struct page *alloc_frozen_pages_noprof(gfp_t, unsigned int order);
#else
static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order)
{
        return __alloc_frozen_pages_noprof(gfp, order, numa_node_id(), NULL);
}
#endif

#define alloc_frozen_pages(...) \
        alloc_hooks(alloc_frozen_pages_noprof(__VA_ARGS__))

extern void zone_pcp_reset(struct zone *zone);
extern void zone_pcp_disable(struct zone *zone);
extern void zone_pcp_enable(struct zone *zone);
extern void zone_pcp_init(struct zone *zone);

extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,
                          phys_addr_t min_addr,
                          int nid, bool exact_nid);

void memmap_init_range(unsigned long, int, unsigned long, unsigned long,
                unsigned long, enum meminit_context, struct vmem_altmap *, int);

#if defined CONFIG_COMPACTION || defined CONFIG_CMA

/*
 * in mm/compaction.c
 */
/*
 * compact_control is used to track pages being migrated and the free pages
 * they are being migrated to during memory compaction. The free_pfn starts
 * at the end of a zone and migrate_pfn begins at the start. Movable pages
 * are moved to the end of a zone during a compaction run and the run
 * completes when free_pfn <= migrate_pfn
 */
struct compact_control {
        struct list_head freepages[NR_PAGE_ORDERS];        /* List of free pages to migrate to */
        struct list_head migratepages;        /* List of pages being migrated */
        unsigned int nr_freepages;        /* Number of isolated free pages */
        unsigned int nr_migratepages;        /* Number of pages to migrate */
        unsigned long free_pfn;                /* isolate_freepages search base */
        /*
         * Acts as an in/out parameter to page isolation for migration.
         * isolate_migratepages uses it as a search base.
         * isolate_migratepages_block will update the value to the next pfn
         * after the last isolated one.
         */
        unsigned long migrate_pfn;
        unsigned long fast_start_pfn;        /* a pfn to start linear scan from */
        struct zone *zone;
        unsigned long total_migrate_scanned;
        unsigned long total_free_scanned;
        unsigned short fast_search_fail;/* failures to use free list searches */
        short search_order;                /* order to start a fast search at */
        const gfp_t gfp_mask;                /* gfp mask of a direct compactor */
        int order;                        /* order a direct compactor needs */
        int migratetype;                /* migratetype of direct compactor */
        const unsigned int alloc_flags;        /* alloc flags of a direct compactor */
        const int highest_zoneidx;        /* zone index of a direct compactor */
        enum migrate_mode mode;                /* Async or sync migration mode */
        bool ignore_skip_hint;                /* Scan blocks even if marked skip */
        bool no_set_skip_hint;                /* Don't mark blocks for skipping */
        bool ignore_block_suitable;        /* Scan blocks considered unsuitable */
        bool direct_compaction;                /* False from kcompactd or /proc/... */
        bool proactive_compaction;        /* kcompactd proactive compaction */
        bool whole_zone;                /* Whole zone should/has been scanned */
        bool contended;                        /* Signal lock contention */
        bool finish_pageblock;                /* Scan the remainder of a pageblock. Used
                                         * when there are potentially transient
                                         * isolation or migration failures to
                                         * ensure forward progress.
                                         */
        bool alloc_contig;                /* alloc_contig_range allocation */
};

/*
 * Used in direct compaction when a page should be taken from the freelists
 * immediately when one is created during the free path.
 */
struct capture_control {
        struct compact_control *cc;
        struct page *page;
};

unsigned long
isolate_freepages_range(struct compact_control *cc,
                        unsigned long start_pfn, unsigned long end_pfn);
int
isolate_migratepages_range(struct compact_control *cc,
                           unsigned long low_pfn, unsigned long end_pfn);

/* Free whole pageblock and set its migration type to MIGRATE_CMA. */
void init_cma_reserved_pageblock(struct page *page);

#endif /* CONFIG_COMPACTION || CONFIG_CMA */

struct cma;

#ifdef CONFIG_CMA
void *cma_reserve_early(struct cma *cma, unsigned long size);
void init_cma_pageblock(struct page *page);
#else
static inline void *cma_reserve_early(struct cma *cma, unsigned long size)
{
        return NULL;
}
static inline void init_cma_pageblock(struct page *page)
{
}
#endif


int find_suitable_fallback(struct free_area *area, unsigned int order,
                           int migratetype, bool claimable);

static inline bool free_area_empty(struct free_area *area, int migratetype)
{
        return list_empty(&area->free_list[migratetype]);
}

/* mm/util.c */
struct anon_vma *folio_anon_vma(const struct folio *folio);

#ifdef CONFIG_MMU
void unmap_mapping_folio(struct folio *folio);
extern long populate_vma_page_range(struct vm_area_struct *vma,
                unsigned long start, unsigned long end, int *locked);
extern long faultin_page_range(struct mm_struct *mm, unsigned long start,
                unsigned long end, bool write, int *locked);
extern bool mlock_future_ok(struct mm_struct *mm, unsigned long flags,
                               unsigned long bytes);

/*
 * NOTE: This function can't tell whether the folio is "fully mapped" in the
 * range.
 * "fully mapped" means all the pages of folio is associated with the page
 * table of range while this function just check whether the folio range is
 * within the range [start, end). Function caller needs to do page table
 * check if it cares about the page table association.
 *
 * Typical usage (like mlock or madvise) is:
 * Caller knows at least 1 page of folio is associated with page table of VMA
 * and the range [start, end) is intersect with the VMA range. Caller wants
 * to know whether the folio is fully associated with the range. It calls
 * this function to check whether the folio is in the range first. Then checks
 * the page table to know whether the folio is fully mapped to the range.
 */
static inline bool
folio_within_range(struct folio *folio, struct vm_area_struct *vma,
                unsigned long start, unsigned long end)
{
        pgoff_t pgoff, addr;
        unsigned long vma_pglen = vma_pages(vma);

        VM_WARN_ON_FOLIO(folio_test_ksm(folio), folio);
        if (start > end)
                return false;

        if (start < vma->vm_start)
                start = vma->vm_start;

        if (end > vma->vm_end)
                end = vma->vm_end;

        pgoff = folio_pgoff(folio);

        /* if folio start address is not in vma range */
        if (!in_range(pgoff, vma->vm_pgoff, vma_pglen))
                return false;

        addr = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);

        return !(addr < start || end - addr < folio_size(folio));
}

static inline bool
folio_within_vma(struct folio *folio, struct vm_area_struct *vma)
{
        return folio_within_range(folio, vma, vma->vm_start, vma->vm_end);
}

/*
 * mlock_vma_folio() and munlock_vma_folio():
 * should be called with vma's mmap_lock held for read or write,
 * under page table lock for the pte/pmd being added or removed.
 *
 * mlock is usually called at the end of folio_add_*_rmap_*(), munlock at
 * the end of folio_remove_rmap_*(); but new anon folios are managed by
 * folio_add_lru_vma() calling mlock_new_folio().
 */
void mlock_folio(struct folio *folio);
static inline void mlock_vma_folio(struct folio *folio,
                                struct vm_area_struct *vma)
{
        /*
         * The VM_SPECIAL check here serves two purposes.
         * 1) VM_IO check prevents migration from double-counting during mlock.
         * 2) Although mmap_region() and mlock_fixup() take care that VM_LOCKED
         *    is never left set on a VM_SPECIAL vma, there is an interval while
         *    file->f_op->mmap() is using vm_insert_page(s), when VM_LOCKED may
         *    still be set while VM_SPECIAL bits are added: so ignore it then.
         */
        if (unlikely((vma->vm_flags & (VM_LOCKED|VM_SPECIAL)) == VM_LOCKED))
                mlock_folio(folio);
}

void munlock_folio(struct folio *folio);
static inline void munlock_vma_folio(struct folio *folio,
                                        struct vm_area_struct *vma)
{
        /*
         * munlock if the function is called. Ideally, we should only
         * do munlock if any page of folio is unmapped from VMA and
         * cause folio not fully mapped to VMA.
         *
         * But it's not easy to confirm that's the situation. So we
         * always munlock the folio and page reclaim will correct it
         * if it's wrong.
         */
        if (unlikely(vma->vm_flags & VM_LOCKED))
                munlock_folio(folio);
}

void mlock_new_folio(struct folio *folio);
bool need_mlock_drain(int cpu);
void mlock_drain_local(void);
void mlock_drain_remote(int cpu);

extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);

/**
 * vma_address - Find the virtual address a page range is mapped at
 * @vma: The vma which maps this object.
 * @pgoff: The page offset within its object.
 * @nr_pages: The number of pages to consider.
 *
 * If any page in this range is mapped by this VMA, return the first address
 * where any of these pages appear.  Otherwise, return -EFAULT.
 */
static inline unsigned long vma_address(const struct vm_area_struct *vma,
                pgoff_t pgoff, unsigned long nr_pages)
{
        unsigned long address;

        if (pgoff >= vma->vm_pgoff) {
                address = vma->vm_start +
                        ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
                /* Check for address beyond vma (or wrapped through 0?) */
                if (address < vma->vm_start || address >= vma->vm_end)
                        address = -EFAULT;
        } else if (pgoff + nr_pages - 1 >= vma->vm_pgoff) {
                /* Test above avoids possibility of wrap to 0 on 32-bit */
                address = vma->vm_start;
        } else {
                address = -EFAULT;
        }
        return address;
}

/*
 * Then at what user virtual address will none of the range be found in vma?
 * Assumes that vma_address() already returned a good starting address.
 */
static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw)
{
        struct vm_area_struct *vma = pvmw->vma;
        pgoff_t pgoff;
        unsigned long address;

        /* Common case, plus ->pgoff is invalid for KSM */
        if (pvmw->nr_pages == 1)
                return pvmw->address + PAGE_SIZE;

        pgoff = pvmw->pgoff + pvmw->nr_pages;
        address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
        /* Check for address beyond vma (or wrapped through 0?) */
        if (address < vma->vm_start || address > vma->vm_end)
                address = vma->vm_end;
        return address;
}

static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
                                                    struct file *fpin)
{
        int flags = vmf->flags;

        if (fpin)
                return fpin;

        /*
         * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
         * anything, so we only pin the file and drop the mmap_lock if only
         * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt.
         */
        if (fault_flag_allow_retry_first(flags) &&
            !(flags & FAULT_FLAG_RETRY_NOWAIT)) {
                fpin = get_file(vmf->vma->vm_file);
                release_fault_lock(vmf);
        }
        return fpin;
}
#else /* !CONFIG_MMU */
static inline void unmap_mapping_folio(struct folio *folio) { }
static inline void mlock_new_folio(struct folio *folio) { }
static inline bool need_mlock_drain(int cpu) { return false; }
static inline void mlock_drain_local(void) { }
static inline void mlock_drain_remote(int cpu) { }
static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
{
}
#endif /* !CONFIG_MMU */

/* Memory initialisation debug and verification */
#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
DECLARE_STATIC_KEY_TRUE(deferred_pages);

bool __init deferred_grow_zone(struct zone *zone, unsigned int order);
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */

void init_deferred_page(unsigned long pfn, int nid);

enum mminit_level {
        MMINIT_WARNING,
        MMINIT_VERIFY,
        MMINIT_TRACE
};

#ifdef CONFIG_DEBUG_MEMORY_INIT

extern int mminit_loglevel;

#define mminit_dprintk(level, prefix, fmt, arg...) \
do { \
        if (level < mminit_loglevel) { \
                if (level <= MMINIT_WARNING) \
                        pr_warn("mminit::" prefix " " fmt, ##arg);        \
                else \
                        printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
        } \
} while (0)

extern void mminit_verify_pageflags_layout(void);
extern void mminit_verify_zonelist(void);
#else

static inline void mminit_dprintk(enum mminit_level level,
                                const char *prefix, const char *fmt, ...)
{
}

static inline void mminit_verify_pageflags_layout(void)
{
}

static inline void mminit_verify_zonelist(void)
{
}
#endif /* CONFIG_DEBUG_MEMORY_INIT */

#define NODE_RECLAIM_NOSCAN        -2
#define NODE_RECLAIM_FULL        -1
#define NODE_RECLAIM_SOME        0
#define NODE_RECLAIM_SUCCESS        1

#ifdef CONFIG_NUMA
extern int node_reclaim_mode;

extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
extern int find_next_best_node(int node, nodemask_t *used_node_mask);
#else
#define node_reclaim_mode 0

static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
                                unsigned int order)
{
        return NODE_RECLAIM_NOSCAN;
}
static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
{
        return NUMA_NO_NODE;
}
#endif

static inline bool node_reclaim_enabled(void)
{
        /* Is any node_reclaim_mode bit set? */
        return node_reclaim_mode & (RECLAIM_ZONE|RECLAIM_WRITE|RECLAIM_UNMAP);
}

/*
 * mm/memory-failure.c
 */
#ifdef CONFIG_MEMORY_FAILURE
int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill);
void shake_folio(struct folio *folio);
extern int hwpoison_filter(struct page *p);

extern u32 hwpoison_filter_dev_major;
extern u32 hwpoison_filter_dev_minor;
extern u64 hwpoison_filter_flags_mask;
extern u64 hwpoison_filter_flags_value;
extern u64 hwpoison_filter_memcg;
extern u32 hwpoison_filter_enable;
#define MAGIC_HWPOISON        0x48575053U        /* HWPS */
void SetPageHWPoisonTakenOff(struct page *page);
void ClearPageHWPoisonTakenOff(struct page *page);
bool take_page_off_buddy(struct page *page);
bool put_page_back_buddy(struct page *page);
struct task_struct *task_early_kill(struct task_struct *tsk, int force_early);
void add_to_kill_ksm(struct task_struct *tsk, const struct page *p,
                     struct vm_area_struct *vma, struct list_head *to_kill,
                     unsigned long ksm_addr);
unsigned long page_mapped_in_vma(const struct page *page,
                struct vm_area_struct *vma);

#else
static inline int unmap_poisoned_folio(struct folio *folio, unsigned long pfn, bool must_kill)
{
        return -EBUSY;
}
#endif

extern unsigned long  __must_check vm_mmap_pgoff(struct file *, unsigned long,
        unsigned long, unsigned long,
        unsigned long, unsigned long);

extern void set_pageblock_order(void);
struct folio *alloc_migrate_folio(struct folio *src, unsigned long private);
unsigned long reclaim_pages(struct list_head *folio_list);
unsigned int reclaim_clean_pages_from_list(struct zone *zone,
                                            struct list_head *folio_list);
/* The ALLOC_WMARK bits are used as an index to zone->watermark */
#define ALLOC_WMARK_MIN                WMARK_MIN
#define ALLOC_WMARK_LOW                WMARK_LOW
#define ALLOC_WMARK_HIGH        WMARK_HIGH
#define ALLOC_NO_WATERMARKS        0x04 /* don't check watermarks at all */

/* Mask to get the watermark bits */
#define ALLOC_WMARK_MASK        (ALLOC_NO_WATERMARKS-1)

/*
 * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
 * cannot assume a reduced access to memory reserves is sufficient for
 * !MMU
 */
#ifdef CONFIG_MMU
#define ALLOC_OOM                0x08
#else
#define ALLOC_OOM                ALLOC_NO_WATERMARKS
#endif

#define ALLOC_NON_BLOCK                 0x10 /* Caller cannot block. Allow access
                                       * to 25% of the min watermark or
                                       * 62.5% if __GFP_HIGH is set.
                                       */
#define ALLOC_MIN_RESERVE         0x20 /* __GFP_HIGH set. Allow access to 50%
                                       * of the min watermark.
                                       */
#define ALLOC_CPUSET                 0x40 /* check for correct cpuset */
#define ALLOC_CMA                 0x80 /* allow allocations from CMA areas */
#ifdef CONFIG_ZONE_DMA32
#define ALLOC_NOFRAGMENT        0x100 /* avoid mixing pageblock types */
#else
#define ALLOC_NOFRAGMENT          0x0
#endif
#define ALLOC_HIGHATOMIC        0x200 /* Allows access to MIGRATE_HIGHATOMIC */
#define ALLOC_TRYLOCK                0x400 /* Only use spin_trylock in allocation path */
#define ALLOC_KSWAPD                0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */

/* Flags that allow allocations below the min watermark. */
#define ALLOC_RESERVES (ALLOC_NON_BLOCK|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)

enum ttu_flags;
struct tlbflush_unmap_batch;


/*
 * only for MM internal work items which do not depend on
 * any allocations or locks which might depend on allocations
 */
extern struct workqueue_struct *mm_percpu_wq;

#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
void try_to_unmap_flush(void);
void try_to_unmap_flush_dirty(void);
void flush_tlb_batched_pending(struct mm_struct *mm);
#else
static inline void try_to_unmap_flush(void)
{
}
static inline void try_to_unmap_flush_dirty(void)
{
}
static inline void flush_tlb_batched_pending(struct mm_struct *mm)
{
}
#endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */

extern const struct trace_print_flags pageflag_names[];
extern const struct trace_print_flags vmaflag_names[];
extern const struct trace_print_flags gfpflag_names[];

static inline bool is_migrate_highatomic(enum migratetype migratetype)
{
        return migratetype == MIGRATE_HIGHATOMIC;
}

void setup_zone_pageset(struct zone *zone);

struct migration_target_control {
        int nid;                /* preferred node id */
        nodemask_t *nmask;
        gfp_t gfp_mask;
        enum migrate_reason reason;
};

/*
 * mm/filemap.c
 */
size_t splice_folio_into_pipe(struct pipe_inode_info *pipe,
                              struct folio *folio, loff_t fpos, size_t size);

/*
 * mm/vmalloc.c
 */
#ifdef CONFIG_MMU
void __init vmalloc_init(void);
int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
                pgprot_t prot, struct page **pages, unsigned int page_shift);
unsigned int get_vm_area_page_order(struct vm_struct *vm);
#else
static inline void vmalloc_init(void)
{
}

static inline
int __must_check vmap_pages_range_noflush(unsigned long addr, unsigned long end,
                pgprot_t prot, struct page **pages, unsigned int page_shift)
{
        return -EINVAL;
}
#endif

int __must_check __vmap_pages_range_noflush(unsigned long addr,
                               unsigned long end, pgprot_t prot,
                               struct page **pages, unsigned int page_shift);

void vunmap_range_noflush(unsigned long start, unsigned long end);

void __vunmap_range_noflush(unsigned long start, unsigned long end);

int numa_migrate_check(struct folio *folio, struct vm_fault *vmf,
                      unsigned long addr, int *flags, bool writable,
                      int *last_cpupid);

void free_zone_device_folio(struct folio *folio);
int migrate_device_coherent_folio(struct folio *folio);

struct vm_struct *__get_vm_area_node(unsigned long size,
                                     unsigned long align, unsigned long shift,
                                     unsigned long flags, unsigned long start,
                                     unsigned long end, int node, gfp_t gfp_mask,
                                     const void *caller);

/*
 * mm/gup.c
 */
int __must_check try_grab_folio(struct folio *folio, int refs,
                                unsigned int flags);

/*
 * mm/huge_memory.c
 */
void touch_pud(struct vm_area_struct *vma, unsigned long addr,
               pud_t *pud, bool write);
void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
               pmd_t *pmd, bool write);

/*
 * Parses a string with mem suffixes into its order. Useful to parse kernel
 * parameters.
 */
static inline int get_order_from_str(const char *size_str,
                                     unsigned long valid_orders)
{
        unsigned long size;
        char *endptr;
        int order;

        size = memparse(size_str, &endptr);

        if (!is_power_of_2(size))
                return -EINVAL;
        order = get_order(size);
        if (BIT(order) & ~valid_orders)
                return -EINVAL;

        return order;
}

enum {
        /* mark page accessed */
        FOLL_TOUCH = 1 << 16,
        /* a retry, previous pass started an IO */
        FOLL_TRIED = 1 << 17,
        /* we are working on non-current tsk/mm */
        FOLL_REMOTE = 1 << 18,
        /* pages must be released via unpin_user_page */
        FOLL_PIN = 1 << 19,
        /* gup_fast: prevent fall-back to slow gup */
        FOLL_FAST_ONLY = 1 << 20,
        /* allow unlocking the mmap lock */
        FOLL_UNLOCKABLE = 1 << 21,
        /* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */
        FOLL_MADV_POPULATE = 1 << 22,
};

#define INTERNAL_GUP_FLAGS (FOLL_TOUCH | FOLL_TRIED | FOLL_REMOTE | FOLL_PIN | \
                            FOLL_FAST_ONLY | FOLL_UNLOCKABLE | \
                            FOLL_MADV_POPULATE)

/*
 * Indicates for which pages that are write-protected in the page table,
 * whether GUP has to trigger unsharing via FAULT_FLAG_UNSHARE such that the
 * GUP pin will remain consistent with the pages mapped into the page tables
 * of the MM.
 *
 * Temporary unmapping of PageAnonExclusive() pages or clearing of
 * PageAnonExclusive() has to protect against concurrent GUP:
 * * Ordinary GUP: Using the PT lock
 * * GUP-fast and fork(): mm->write_protect_seq
 * * GUP-fast and KSM or temporary unmapping (swap, migration): see
 *    folio_try_share_anon_rmap_*()
 *
 * Must be called with the (sub)page that's actually referenced via the
 * page table entry, which might not necessarily be the head page for a
 * PTE-mapped THP.
 *
 * If the vma is NULL, we're coming from the GUP-fast path and might have
 * to fallback to the slow path just to lookup the vma.
 */
static inline bool gup_must_unshare(struct vm_area_struct *vma,
                                    unsigned int flags, struct page *page)
{
        /*
         * FOLL_WRITE is implicitly handled correctly as the page table entry
         * has to be writable -- and if it references (part of) an anonymous
         * folio, that part is required to be marked exclusive.
         */
        if ((flags & (FOLL_WRITE | FOLL_PIN)) != FOLL_PIN)
                return false;
        /*
         * Note: PageAnon(page) is stable until the page is actually getting
         * freed.
         */
        if (!PageAnon(page)) {
                /*
                 * We only care about R/O long-term pining: R/O short-term
                 * pinning does not have the semantics to observe successive
                 * changes through the process page tables.
                 */
                if (!(flags & FOLL_LONGTERM))
                        return false;

                /* We really need the vma ... */
                if (!vma)
                        return true;

                /*
                 * ... because we only care about writable private ("COW")
                 * mappings where we have to break COW early.
                 */
                return is_cow_mapping(vma->vm_flags);
        }

        /* Paired with a memory barrier in folio_try_share_anon_rmap_*(). */
        if (IS_ENABLED(CONFIG_HAVE_GUP_FAST))
                smp_rmb();

        /*
         * Note that KSM pages cannot be exclusive, and consequently,
         * cannot get pinned.
         */
        return !PageAnonExclusive(page);
}

extern bool mirrored_kernelcore;
bool memblock_has_mirror(void);
void memblock_free_all(void);

static __always_inline void vma_set_range(struct vm_area_struct *vma,
                                          unsigned long start, unsigned long end,
                                          pgoff_t pgoff)
{
        vma->vm_start = start;
        vma->vm_end = end;
        vma->vm_pgoff = pgoff;
}

static inline bool vma_soft_dirty_enabled(struct vm_area_struct *vma)
{
        /*
         * NOTE: we must check this before VM_SOFTDIRTY on soft-dirty
         * enablements, because when without soft-dirty being compiled in,
         * VM_SOFTDIRTY is defined as 0x0, then !(vm_flags & VM_SOFTDIRTY)
         * will be constantly true.
         */
        if (!IS_ENABLED(CONFIG_MEM_SOFT_DIRTY))
                return false;

        /*
         * Soft-dirty is kind of special: its tracking is enabled when the
         * vma flags not set.
         */
        return !(vma->vm_flags & VM_SOFTDIRTY);
}

static inline bool pmd_needs_soft_dirty_wp(struct vm_area_struct *vma, pmd_t pmd)
{
        return vma_soft_dirty_enabled(vma) && !pmd_soft_dirty(pmd);
}

static inline bool pte_needs_soft_dirty_wp(struct vm_area_struct *vma, pte_t pte)
{
        return vma_soft_dirty_enabled(vma) && !pte_soft_dirty(pte);
}

void __meminit __init_single_page(struct page *page, unsigned long pfn,
                                unsigned long zone, int nid);
void __meminit __init_page_from_nid(unsigned long pfn, int nid);

/* shrinker related functions */
unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
                          int priority);

#ifdef CONFIG_SHRINKER_DEBUG
static inline __printf(2, 0) int shrinker_debugfs_name_alloc(
                        struct shrinker *shrinker, const char *fmt, va_list ap)
{
        shrinker->name = kvasprintf_const(GFP_KERNEL, fmt, ap);

        return shrinker->name ? 0 : -ENOMEM;
}

static inline void shrinker_debugfs_name_free(struct shrinker *shrinker)
{
        kfree_const(shrinker->name);
        shrinker->name = NULL;
}

extern int shrinker_debugfs_add(struct shrinker *shrinker);
extern struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker,
                                              int *debugfs_id);
extern void shrinker_debugfs_remove(struct dentry *debugfs_entry,
                                    int debugfs_id);
#else /* CONFIG_SHRINKER_DEBUG */
static inline int shrinker_debugfs_add(struct shrinker *shrinker)
{
        return 0;
}
static inline int shrinker_debugfs_name_alloc(struct shrinker *shrinker,
                                              const char *fmt, va_list ap)
{
        return 0;
}
static inline void shrinker_debugfs_name_free(struct shrinker *shrinker)
{
}
static inline struct dentry *shrinker_debugfs_detach(struct shrinker *shrinker,
                                                     int *debugfs_id)
{
        *debugfs_id = -1;
        return NULL;
}
static inline void shrinker_debugfs_remove(struct dentry *debugfs_entry,
                                           int debugfs_id)
{
}
#endif /* CONFIG_SHRINKER_DEBUG */

/* Only track the nodes of mappings with shadow entries */
void workingset_update_node(struct xa_node *node);
extern struct list_lru shadow_nodes;
#define mapping_set_update(xas, mapping) do {                        \
        if (!dax_mapping(mapping) && !shmem_mapping(mapping)) {        \
                xas_set_update(xas, workingset_update_node);        \
                xas_set_lru(xas, &shadow_nodes);                \
        }                                                        \
} while (0)

/* mremap.c */
unsigned long move_page_tables(struct pagetable_move_control *pmc);

#ifdef CONFIG_UNACCEPTED_MEMORY
void accept_page(struct page *page);
#else /* CONFIG_UNACCEPTED_MEMORY */
static inline void accept_page(struct page *page)
{
}
#endif /* CONFIG_UNACCEPTED_MEMORY */

/* pagewalk.c */
int walk_page_range_mm(struct mm_struct *mm, unsigned long start,
                unsigned long end, const struct mm_walk_ops *ops,
                void *private);
int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
                          unsigned long end, const struct mm_walk_ops *ops,
                          pgd_t *pgd, void *private);

/* pt_reclaim.c */
bool try_get_and_clear_pmd(struct mm_struct *mm, pmd_t *pmd, pmd_t *pmdval);
void free_pte(struct mm_struct *mm, unsigned long addr, struct mmu_gather *tlb,
              pmd_t pmdval);
void try_to_free_pte(struct mm_struct *mm, pmd_t *pmd, unsigned long addr,
                     struct mmu_gather *tlb);

#ifdef CONFIG_PT_RECLAIM
bool reclaim_pt_is_enabled(unsigned long start, unsigned long end,
                           struct zap_details *details);
#else
static inline bool reclaim_pt_is_enabled(unsigned long start, unsigned long end,
                                         struct zap_details *details)
{
        return false;
}
#endif /* CONFIG_PT_RECLAIM */

void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm);
int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm);

#endif        /* __MM_INTERNAL_H */


















































  254 



















































































































































































   22 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 */

#include <linux/init.h>
#include <linux/usb.h>

#include <sound/core.h>
#include <sound/info.h>
#include <sound/pcm.h>

#include "usbaudio.h"
#include "helper.h"
#include "card.h"
#include "endpoint.h"
#include "proc.h"

/* convert our full speed USB rate into sampling rate in Hz */
static inline unsigned get_full_speed_hz(unsigned int usb_rate)
{
        return (usb_rate * 125 + (1 << 12)) >> 13;
}

/* convert our high speed USB rate into sampling rate in Hz */
static inline unsigned get_high_speed_hz(unsigned int usb_rate)
{
        return (usb_rate * 125 + (1 << 9)) >> 10;
}

/*
 * common proc files to show the usb device info
 */
static void proc_audio_usbbus_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer)
{
        struct snd_usb_audio *chip = entry->private_data;
        if (!atomic_read(&chip->shutdown))
                snd_iprintf(buffer, "%03d/%03d\n", chip->dev->bus->busnum, chip->dev->devnum);
}

static void proc_audio_usbid_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer)
{
        struct snd_usb_audio *chip = entry->private_data;
        if (!atomic_read(&chip->shutdown))
                snd_iprintf(buffer, "%04x:%04x\n", 
                            USB_ID_VENDOR(chip->usb_id),
                            USB_ID_PRODUCT(chip->usb_id));
}

void snd_usb_audio_create_proc(struct snd_usb_audio *chip)
{
        snd_card_ro_proc_new(chip->card, "usbbus", chip,
                             proc_audio_usbbus_read);
        snd_card_ro_proc_new(chip->card, "usbid", chip,
                             proc_audio_usbid_read);
}

static const char * const channel_labels[] = {
        [SNDRV_CHMAP_NA]        = "N/A",
        [SNDRV_CHMAP_MONO]        = "MONO",
        [SNDRV_CHMAP_FL]        = "FL",
        [SNDRV_CHMAP_FR]        = "FR",
        [SNDRV_CHMAP_FC]        = "FC",
        [SNDRV_CHMAP_LFE]        = "LFE",
        [SNDRV_CHMAP_RL]        = "RL",
        [SNDRV_CHMAP_RR]        = "RR",
        [SNDRV_CHMAP_FLC]        = "FLC",
        [SNDRV_CHMAP_FRC]        = "FRC",
        [SNDRV_CHMAP_RC]        = "RC",
        [SNDRV_CHMAP_SL]        = "SL",
        [SNDRV_CHMAP_SR]        = "SR",
        [SNDRV_CHMAP_TC]        = "TC",
        [SNDRV_CHMAP_TFL]        = "TFL",
        [SNDRV_CHMAP_TFC]        = "TFC",
        [SNDRV_CHMAP_TFR]        = "TFR",
        [SNDRV_CHMAP_TRL]        = "TRL",
        [SNDRV_CHMAP_TRC]        = "TRC",
        [SNDRV_CHMAP_TRR]        = "TRR",
        [SNDRV_CHMAP_TFLC]        = "TFLC",
        [SNDRV_CHMAP_TFRC]        = "TFRC",
        [SNDRV_CHMAP_LLFE]        = "LLFE",
        [SNDRV_CHMAP_RLFE]        = "RLFE",
        [SNDRV_CHMAP_TSL]        = "TSL",
        [SNDRV_CHMAP_TSR]        = "TSR",
        [SNDRV_CHMAP_BC]        = "BC",
        [SNDRV_CHMAP_RLC]        = "RLC",
        [SNDRV_CHMAP_RRC]        = "RRC",
};

/*
 * proc interface for list the supported pcm formats
 */
static void proc_dump_substream_formats(struct snd_usb_substream *subs, struct snd_info_buffer *buffer)
{
        struct audioformat *fp;
        static const char * const sync_types[4] = {
                "NONE", "ASYNC", "ADAPTIVE", "SYNC"
        };

        list_for_each_entry(fp, &subs->fmt_list, list) {
                snd_pcm_format_t fmt;

                snd_iprintf(buffer, "  Interface %d\n", fp->iface);
                snd_iprintf(buffer, "    Altset %d\n", fp->altsetting);
                snd_iprintf(buffer, "    Format:");
                pcm_for_each_format(fmt)
                        if (fp->formats & pcm_format_to_bits(fmt))
                                snd_iprintf(buffer, " %s",
                                            snd_pcm_format_name(fmt));
                snd_iprintf(buffer, "\n");
                snd_iprintf(buffer, "    Channels: %d\n", fp->channels);
                snd_iprintf(buffer, "    Endpoint: 0x%02x (%d %s) (%s)\n",
                            fp->endpoint,
                            fp->endpoint & USB_ENDPOINT_NUMBER_MASK,
                            fp->endpoint & USB_DIR_IN ? "IN" : "OUT",
                            sync_types[(fp->ep_attr & USB_ENDPOINT_SYNCTYPE) >> 2]);
                if (fp->rates & SNDRV_PCM_RATE_CONTINUOUS) {
                        snd_iprintf(buffer, "    Rates: %d - %d (continuous)\n",
                                    fp->rate_min, fp->rate_max);
                } else {
                        unsigned int i;
                        snd_iprintf(buffer, "    Rates: ");
                        for (i = 0; i < fp->nr_rates; i++) {
                                if (i > 0)
                                        snd_iprintf(buffer, ", ");
                                snd_iprintf(buffer, "%d", fp->rate_table[i]);
                        }
                        snd_iprintf(buffer, "\n");
                }
                if (subs->speed != USB_SPEED_FULL)
                        snd_iprintf(buffer, "    Data packet interval: %d us\n",
                                    125 * (1 << fp->datainterval));
                snd_iprintf(buffer, "    Bits: %d\n", fp->fmt_bits);

                if (fp->dsd_raw)
                        snd_iprintf(buffer, "    DSD raw: DOP=%d, bitrev=%d\n",
                                    fp->dsd_dop, fp->dsd_bitrev);

                if (fp->chmap) {
                        const struct snd_pcm_chmap_elem *map = fp->chmap;
                        int c;

                        snd_iprintf(buffer, "    Channel map:");
                        for (c = 0; c < map->channels; c++) {
                                if (map->map[c] >= ARRAY_SIZE(channel_labels) ||
                                    !channel_labels[map->map[c]])
                                        snd_iprintf(buffer, " --");
                                else
                                        snd_iprintf(buffer, " %s",
                                                    channel_labels[map->map[c]]);
                        }
                        snd_iprintf(buffer, "\n");
                }

                if (fp->sync_ep) {
                        snd_iprintf(buffer, "    Sync Endpoint: 0x%02x (%d %s)\n",
                                    fp->sync_ep,
                                    fp->sync_ep & USB_ENDPOINT_NUMBER_MASK,
                                    fp->sync_ep & USB_DIR_IN ? "IN" : "OUT");
                        snd_iprintf(buffer, "    Sync EP Interface: %d\n",
                                    fp->sync_iface);
                        snd_iprintf(buffer, "    Sync EP Altset: %d\n",
                                    fp->sync_altsetting);
                        snd_iprintf(buffer, "    Implicit Feedback Mode: %s\n",
                                    fp->implicit_fb ? "Yes" : "No");
                }

                // snd_iprintf(buffer, "    Max Packet Size = %d\n", fp->maxpacksize);
                // snd_iprintf(buffer, "    EP Attribute = %#x\n", fp->attributes);
        }
}

static void proc_dump_ep_status(struct snd_usb_substream *subs,
                                struct snd_usb_endpoint *data_ep,
                                struct snd_usb_endpoint *sync_ep,
                                struct snd_info_buffer *buffer)
{
        if (!data_ep)
                return;
        snd_iprintf(buffer, "    Packet Size = %d\n", data_ep->curpacksize);
        snd_iprintf(buffer, "    Momentary freq = %u Hz (%#x.%04x)\n",
                    subs->speed == USB_SPEED_FULL
                    ? get_full_speed_hz(data_ep->freqm)
                    : get_high_speed_hz(data_ep->freqm),
                    data_ep->freqm >> 16, data_ep->freqm & 0xffff);
        if (sync_ep && data_ep->freqshift != INT_MIN) {
                int res = 16 - data_ep->freqshift;
                snd_iprintf(buffer, "    Feedback Format = %d.%d\n",
                            (sync_ep->syncmaxsize > 3 ? 32 : 24) - res, res);
        }
}

static void proc_dump_substream_status(struct snd_usb_audio *chip,
                                       struct snd_usb_substream *subs,
                                       struct snd_info_buffer *buffer)
{
        mutex_lock(&chip->mutex);
        if (subs->running) {
                snd_iprintf(buffer, "  Status: Running\n");
                if (subs->cur_audiofmt) {
                        snd_iprintf(buffer, "    Interface = %d\n", subs->cur_audiofmt->iface);
                        snd_iprintf(buffer, "    Altset = %d\n", subs->cur_audiofmt->altsetting);
                }
                proc_dump_ep_status(subs, subs->data_endpoint, subs->sync_endpoint, buffer);
        } else {
                snd_iprintf(buffer, "  Status: Stop\n");
        }
        mutex_unlock(&chip->mutex);
}

static void proc_pcm_format_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer)
{
        struct snd_usb_stream *stream = entry->private_data;
        struct snd_usb_audio *chip = stream->chip;

        snd_iprintf(buffer, "%s : %s\n", chip->card->longname, stream->pcm->name);

        if (stream->substream[SNDRV_PCM_STREAM_PLAYBACK].num_formats) {
                snd_iprintf(buffer, "\nPlayback:\n");
                proc_dump_substream_status(chip, &stream->substream[SNDRV_PCM_STREAM_PLAYBACK], buffer);
                proc_dump_substream_formats(&stream->substream[SNDRV_PCM_STREAM_PLAYBACK], buffer);
        }
        if (stream->substream[SNDRV_PCM_STREAM_CAPTURE].num_formats) {
                snd_iprintf(buffer, "\nCapture:\n");
                proc_dump_substream_status(chip, &stream->substream[SNDRV_PCM_STREAM_CAPTURE], buffer);
                proc_dump_substream_formats(&stream->substream[SNDRV_PCM_STREAM_CAPTURE], buffer);
        }
}

void snd_usb_proc_pcm_format_add(struct snd_usb_stream *stream)
{
        char name[32];
        struct snd_card *card = stream->chip->card;

        sprintf(name, "stream%d", stream->pcm_index);
        snd_card_ro_proc_new(card, name, stream, proc_pcm_format_read);
}
































































    5 

   17 

   17 















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_SCHED_RCUPDATE_WAIT_H
#define _LINUX_SCHED_RCUPDATE_WAIT_H

/*
 * RCU synchronization types and methods:
 */

#include <linux/rcupdate.h>
#include <linux/completion.h>
#include <linux/sched.h>

/*
 * Structure allowing asynchronous waiting on RCU.
 */
struct rcu_synchronize {
        struct rcu_head head;
        struct completion completion;

        /* This is for debugging. */
        struct rcu_gp_oldstate oldstate;
};
void wakeme_after_rcu(struct rcu_head *head);

void __wait_rcu_gp(bool checktiny, unsigned int state, int n, call_rcu_func_t *crcu_array,
                   struct rcu_synchronize *rs_array);

#define _wait_rcu_gp(checktiny, state, ...) \
do {                                                                                                \
        call_rcu_func_t __crcu_array[] = { __VA_ARGS__ };                                        \
        struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)];                                \
        __wait_rcu_gp(checktiny, state, ARRAY_SIZE(__crcu_array), __crcu_array, __rs_array);        \
} while (0)

#define wait_rcu_gp(...) _wait_rcu_gp(false, TASK_UNINTERRUPTIBLE, __VA_ARGS__)
#define wait_rcu_gp_state(state, ...) _wait_rcu_gp(false, state, __VA_ARGS__)

/**
 * synchronize_rcu_mult - Wait concurrently for multiple grace periods
 * @...: List of call_rcu() functions for different grace periods to wait on
 *
 * This macro waits concurrently for multiple types of RCU grace periods.
 * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait
 * on concurrent RCU and RCU-tasks grace periods.  Waiting on a given SRCU
 * domain requires you to write a wrapper function for that SRCU domain's
 * call_srcu() function, with this wrapper supplying the pointer to the
 * corresponding srcu_struct.
 *
 * Note that call_rcu_hurry() should be used instead of call_rcu()
 * because in kernels built with CONFIG_RCU_LAZY=y the delay between the
 * invocation of call_rcu() and that of the corresponding RCU callback
 * can be multiple seconds.
 *
 * The first argument tells Tiny RCU's _wait_rcu_gp() not to
 * bother waiting for RCU.  The reason for this is because anywhere
 * synchronize_rcu_mult() can be called is automatically already a full
 * grace period.
 */
#define synchronize_rcu_mult(...) \
        _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), TASK_UNINTERRUPTIBLE, __VA_ARGS__)

static inline void cond_resched_rcu(void)
{
#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
        rcu_read_unlock();
        cond_resched();
        rcu_read_lock();
#endif
}

// Has the current task blocked within its current RCU read-side
// critical section?
static inline bool has_rcu_reader_blocked(void)
{
#ifdef CONFIG_PREEMPT_RCU
        return !list_empty(&current->rcu_node_entry);
#else
        return false;
#endif
}

#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */
















































   26 



   16 
    8 








   19 
    5 



   22 







    4 

   18 


















   36 



   13 









   45 






 3270 



 3262 
    9 






   40 






    8 
   40 

   35 














   12 







    6 

    1 



    5 


   11 



   11 
    4 

    7 





   11 






    9 
    8 



    7 




    4 



    3 
    3 




    3 













    2 


    1 
















    6 



    6 

    2 


    4 
    6 















    2 
    2 

    2 



    2 

    3 




































    2 
    1 










































    1 















    2 


    2 

    1 








    2 
    2 









    2 











    2 




    2 










    1 


    3 







    2 
    1 





  275 





  138 

    2 


    4 


    3 


    4 


    2 



    4 


    2 


   26 






    2 


   30 










    1 

   60 









    5 



    7 


    5 


    9 


    3 


    3 


    4 


    3 


   30 


   17 










    1 


    9 








   14 
   13 









    1 


  278 




    2 


  256 
   13 





  271 


  271 











































































































































































































































   34 



   34 
   34 
   32 
    1 
    1 

   34 






   32 






    1 


   28 





















    2 
    1 




















   37 









    3 



   32 

    1 
   31 
   32 


    2 











    1 
    3 



   35 








   27 






    2 





    3 

    1 
    2 
    3 


    3 
    2 





















  100 





   81 
   21 


















   19 









   13 
    1 










   38 




   36 



    5 






































   28 
    1 













  122 

   99 
   28 









   39 











   23 




   16 




   39 




 1286 



 1253 
   39 
   39 
   38 























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/fs/fcntl.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

#include <linux/syscalls.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/sched/task.h>
#include <linux/fs.h>
#include <linux/filelock.h>
#include <linux/file.h>
#include <linux/capability.h>
#include <linux/dnotify.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/pipe_fs_i.h>
#include <linux/security.h>
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/rcupdate.h>
#include <linux/pid_namespace.h>
#include <linux/user_namespace.h>
#include <linux/memfd.h>
#include <linux/compat.h>
#include <linux/mount.h>
#include <linux/rw_hint.h>

#include <linux/poll.h>
#include <asm/siginfo.h>
#include <linux/uaccess.h>

#include "internal.h"

#define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME)

static int setfl(int fd, struct file * filp, unsigned int arg)
{
        struct inode * inode = file_inode(filp);
        int error = 0;

        /*
         * O_APPEND cannot be cleared if the file is marked as append-only
         * and the file is open for write.
         */
        if (((arg ^ filp->f_flags) & O_APPEND) && IS_APPEND(inode))
                return -EPERM;

        /* O_NOATIME can only be set by the owner or superuser */
        if ((arg & O_NOATIME) && !(filp->f_flags & O_NOATIME))
                if (!inode_owner_or_capable(file_mnt_idmap(filp), inode))
                        return -EPERM;

        /* required for strict SunOS emulation */
        if (O_NONBLOCK != O_NDELAY)
               if (arg & O_NDELAY)
                   arg |= O_NONBLOCK;

        /* Pipe packetized mode is controlled by O_DIRECT flag */
        if (!S_ISFIFO(inode->i_mode) &&
            (arg & O_DIRECT) &&
            !(filp->f_mode & FMODE_CAN_ODIRECT))
                return -EINVAL;

        if (filp->f_op->check_flags)
                error = filp->f_op->check_flags(arg);
        if (error)
                return error;

        /*
         * ->fasync() is responsible for setting the FASYNC bit.
         */
        if (((arg ^ filp->f_flags) & FASYNC) && filp->f_op->fasync) {
                error = filp->f_op->fasync(fd, filp, (arg & FASYNC) != 0);
                if (error < 0)
                        goto out;
                if (error > 0)
                        error = 0;
        }
        spin_lock(&filp->f_lock);
        filp->f_flags = (arg & SETFL_MASK) | (filp->f_flags & ~SETFL_MASK);
        filp->f_iocb_flags = iocb_flags(filp);
        spin_unlock(&filp->f_lock);

 out:
        return error;
}

/*
 * Allocate an file->f_owner struct if it doesn't exist, handling racing
 * allocations correctly.
 */
int file_f_owner_allocate(struct file *file)
{
        struct fown_struct *f_owner;

        f_owner = file_f_owner(file);
        if (f_owner)
                return 0;

        f_owner = kzalloc(sizeof(struct fown_struct), GFP_KERNEL);
        if (!f_owner)
                return -ENOMEM;

        rwlock_init(&f_owner->lock);
        f_owner->file = file;
        /* If someone else raced us, drop our allocation. */
        if (unlikely(cmpxchg(&file->f_owner, NULL, f_owner)))
                kfree(f_owner);
        return 0;
}
EXPORT_SYMBOL(file_f_owner_allocate);

void file_f_owner_release(struct file *file)
{
        struct fown_struct *f_owner;

        f_owner = file_f_owner(file);
        if (f_owner) {
                put_pid(f_owner->pid);
                kfree(f_owner);
        }
}

void __f_setown(struct file *filp, struct pid *pid, enum pid_type type,
                int force)
{
        struct fown_struct *f_owner;

        f_owner = file_f_owner(filp);
        if (WARN_ON_ONCE(!f_owner))
                return;

        write_lock_irq(&f_owner->lock);
        if (force || !f_owner->pid) {
                put_pid(f_owner->pid);
                f_owner->pid = get_pid(pid);
                f_owner->pid_type = type;

                if (pid) {
                        const struct cred *cred = current_cred();
                        security_file_set_fowner(filp);
                        f_owner->uid = cred->uid;
                        f_owner->euid = cred->euid;
                }
        }
        write_unlock_irq(&f_owner->lock);
}
EXPORT_SYMBOL(__f_setown);

int f_setown(struct file *filp, int who, int force)
{
        enum pid_type type;
        struct pid *pid = NULL;
        int ret = 0;

        might_sleep();

        type = PIDTYPE_TGID;
        if (who < 0) {
                /* avoid overflow below */
                if (who == INT_MIN)
                        return -EINVAL;

                type = PIDTYPE_PGID;
                who = -who;
        }

        ret = file_f_owner_allocate(filp);
        if (ret)
                return ret;

        rcu_read_lock();
        if (who) {
                pid = find_vpid(who);
                if (!pid)
                        ret = -ESRCH;
        }

        if (!ret)
                __f_setown(filp, pid, type, force);
        rcu_read_unlock();

        return ret;
}
EXPORT_SYMBOL(f_setown);

void f_delown(struct file *filp)
{
        __f_setown(filp, NULL, PIDTYPE_TGID, 1);
}

pid_t f_getown(struct file *filp)
{
        pid_t pid = 0;
        struct fown_struct *f_owner;

        f_owner = file_f_owner(filp);
        if (!f_owner)
                return pid;

        read_lock_irq(&f_owner->lock);
        rcu_read_lock();
        if (pid_task(f_owner->pid, f_owner->pid_type)) {
                pid = pid_vnr(f_owner->pid);
                if (f_owner->pid_type == PIDTYPE_PGID)
                        pid = -pid;
        }
        rcu_read_unlock();
        read_unlock_irq(&f_owner->lock);
        return pid;
}

static int f_setown_ex(struct file *filp, unsigned long arg)
{
        struct f_owner_ex __user *owner_p = (void __user *)arg;
        struct f_owner_ex owner;
        struct pid *pid;
        int type;
        int ret;

        ret = copy_from_user(&owner, owner_p, sizeof(owner));
        if (ret)
                return -EFAULT;

        switch (owner.type) {
        case F_OWNER_TID:
                type = PIDTYPE_PID;
                break;

        case F_OWNER_PID:
                type = PIDTYPE_TGID;
                break;

        case F_OWNER_PGRP:
                type = PIDTYPE_PGID;
                break;

        default:
                return -EINVAL;
        }

        ret = file_f_owner_allocate(filp);
        if (ret)
                return ret;

        rcu_read_lock();
        pid = find_vpid(owner.pid);
        if (owner.pid && !pid)
                ret = -ESRCH;
        else
                 __f_setown(filp, pid, type, 1);
        rcu_read_unlock();

        return ret;
}

static int f_getown_ex(struct file *filp, unsigned long arg)
{
        struct f_owner_ex __user *owner_p = (void __user *)arg;
        struct f_owner_ex owner = {};
        int ret = 0;
        struct fown_struct *f_owner;
        enum pid_type pid_type = PIDTYPE_PID;

        f_owner = file_f_owner(filp);
        if (f_owner) {
                read_lock_irq(&f_owner->lock);
                rcu_read_lock();
                if (pid_task(f_owner->pid, f_owner->pid_type))
                        owner.pid = pid_vnr(f_owner->pid);
                rcu_read_unlock();
                pid_type = f_owner->pid_type;
        }

        switch (pid_type) {
        case PIDTYPE_PID:
                owner.type = F_OWNER_TID;
                break;

        case PIDTYPE_TGID:
                owner.type = F_OWNER_PID;
                break;

        case PIDTYPE_PGID:
                owner.type = F_OWNER_PGRP;
                break;

        default:
                WARN_ON(1);
                ret = -EINVAL;
                break;
        }
        if (f_owner)
                read_unlock_irq(&f_owner->lock);

        if (!ret) {
                ret = copy_to_user(owner_p, &owner, sizeof(owner));
                if (ret)
                        ret = -EFAULT;
        }
        return ret;
}

#ifdef CONFIG_CHECKPOINT_RESTORE
static int f_getowner_uids(struct file *filp, unsigned long arg)
{
        struct user_namespace *user_ns = current_user_ns();
        struct fown_struct *f_owner;
        uid_t __user *dst = (void __user *)arg;
        uid_t src[2] = {0, 0};
        int err;

        f_owner = file_f_owner(filp);
        if (f_owner) {
                read_lock_irq(&f_owner->lock);
                src[0] = from_kuid(user_ns, f_owner->uid);
                src[1] = from_kuid(user_ns, f_owner->euid);
                read_unlock_irq(&f_owner->lock);
        }

        err  = put_user(src[0], &dst[0]);
        err |= put_user(src[1], &dst[1]);

        return err;
}
#else
static int f_getowner_uids(struct file *filp, unsigned long arg)
{
        return -EINVAL;
}
#endif

static bool rw_hint_valid(u64 hint)
{
        BUILD_BUG_ON(WRITE_LIFE_NOT_SET != RWH_WRITE_LIFE_NOT_SET);
        BUILD_BUG_ON(WRITE_LIFE_NONE != RWH_WRITE_LIFE_NONE);
        BUILD_BUG_ON(WRITE_LIFE_SHORT != RWH_WRITE_LIFE_SHORT);
        BUILD_BUG_ON(WRITE_LIFE_MEDIUM != RWH_WRITE_LIFE_MEDIUM);
        BUILD_BUG_ON(WRITE_LIFE_LONG != RWH_WRITE_LIFE_LONG);
        BUILD_BUG_ON(WRITE_LIFE_EXTREME != RWH_WRITE_LIFE_EXTREME);

        switch (hint) {
        case RWH_WRITE_LIFE_NOT_SET:
        case RWH_WRITE_LIFE_NONE:
        case RWH_WRITE_LIFE_SHORT:
        case RWH_WRITE_LIFE_MEDIUM:
        case RWH_WRITE_LIFE_LONG:
        case RWH_WRITE_LIFE_EXTREME:
                return true;
        default:
                return false;
        }
}

static long fcntl_get_rw_hint(struct file *file, unsigned int cmd,
                              unsigned long arg)
{
        struct inode *inode = file_inode(file);
        u64 __user *argp = (u64 __user *)arg;
        u64 hint = READ_ONCE(inode->i_write_hint);

        if (copy_to_user(argp, &hint, sizeof(*argp)))
                return -EFAULT;
        return 0;
}

static long fcntl_set_rw_hint(struct file *file, unsigned int cmd,
                              unsigned long arg)
{
        struct inode *inode = file_inode(file);
        u64 __user *argp = (u64 __user *)arg;
        u64 hint;

        if (!inode_owner_or_capable(file_mnt_idmap(file), inode))
                return -EPERM;

        if (copy_from_user(&hint, argp, sizeof(hint)))
                return -EFAULT;
        if (!rw_hint_valid(hint))
                return -EINVAL;

        WRITE_ONCE(inode->i_write_hint, hint);

        /*
         * file->f_mapping->host may differ from inode. As an example,
         * blkdev_open() modifies file->f_mapping.
         */
        if (file->f_mapping->host != inode)
                WRITE_ONCE(file->f_mapping->host->i_write_hint, hint);

        return 0;
}

/* Is the file descriptor a dup of the file? */
static long f_dupfd_query(int fd, struct file *filp)
{
        CLASS(fd_raw, f)(fd);

        if (fd_empty(f))
                return -EBADF;

        /*
         * We can do the 'fdput()' immediately, as the only thing that
         * matters is the pointer value which isn't changed by the fdput.
         *
         * Technically we didn't need a ref at all, and 'fdget()' was
         * overkill, but given our lockless file pointer lookup, the
         * alternatives are complicated.
         */
        return fd_file(f) == filp;
}

/* Let the caller figure out whether a given file was just created. */
static long f_created_query(const struct file *filp)
{
        return !!(filp->f_mode & FMODE_CREATED);
}

static int f_owner_sig(struct file *filp, int signum, bool setsig)
{
        int ret = 0;
        struct fown_struct *f_owner;

        might_sleep();

        if (setsig) {
                if (!valid_signal(signum))
                        return -EINVAL;

                ret = file_f_owner_allocate(filp);
                if (ret)
                        return ret;
        }

        f_owner = file_f_owner(filp);
        if (setsig)
                f_owner->signum = signum;
        else if (f_owner)
                ret = f_owner->signum;
        return ret;
}

static long do_fcntl(int fd, unsigned int cmd, unsigned long arg,
                struct file *filp)
{
        void __user *argp = (void __user *)arg;
        int argi = (int)arg;
        struct flock flock;
        long err = -EINVAL;

        switch (cmd) {
        case F_CREATED_QUERY:
                err = f_created_query(filp);
                break;
        case F_DUPFD:
                err = f_dupfd(argi, filp, 0);
                break;
        case F_DUPFD_CLOEXEC:
                err = f_dupfd(argi, filp, O_CLOEXEC);
                break;
        case F_DUPFD_QUERY:
                err = f_dupfd_query(argi, filp);
                break;
        case F_GETFD:
                err = get_close_on_exec(fd) ? FD_CLOEXEC : 0;
                break;
        case F_SETFD:
                err = 0;
                set_close_on_exec(fd, argi & FD_CLOEXEC);
                break;
        case F_GETFL:
                err = filp->f_flags;
                break;
        case F_SETFL:
                err = setfl(fd, filp, argi);
                break;
#if BITS_PER_LONG != 32
        /* 32-bit arches must use fcntl64() */
        case F_OFD_GETLK:
#endif
        case F_GETLK:
                if (copy_from_user(&flock, argp, sizeof(flock)))
                        return -EFAULT;
                err = fcntl_getlk(filp, cmd, &flock);
                if (!err && copy_to_user(argp, &flock, sizeof(flock)))
                        return -EFAULT;
                break;
#if BITS_PER_LONG != 32
        /* 32-bit arches must use fcntl64() */
        case F_OFD_SETLK:
        case F_OFD_SETLKW:
                fallthrough;
#endif
        case F_SETLK:
        case F_SETLKW:
                if (copy_from_user(&flock, argp, sizeof(flock)))
                        return -EFAULT;
                err = fcntl_setlk(fd, filp, cmd, &flock);
                break;
        case F_GETOWN:
                /*
                 * XXX If f_owner is a process group, the
                 * negative return value will get converted
                 * into an error.  Oops.  If we keep the
                 * current syscall conventions, the only way
                 * to fix this will be in libc.
                 */
                err = f_getown(filp);
                force_successful_syscall_return();
                break;
        case F_SETOWN:
                err = f_setown(filp, argi, 1);
                break;
        case F_GETOWN_EX:
                err = f_getown_ex(filp, arg);
                break;
        case F_SETOWN_EX:
                err = f_setown_ex(filp, arg);
                break;
        case F_GETOWNER_UIDS:
                err = f_getowner_uids(filp, arg);
                break;
        case F_GETSIG:
                err = f_owner_sig(filp, 0, false);
                break;
        case F_SETSIG:
                err = f_owner_sig(filp, argi, true);
                break;
        case F_GETLEASE:
                err = fcntl_getlease(filp);
                break;
        case F_SETLEASE:
                err = fcntl_setlease(fd, filp, argi);
                break;
        case F_NOTIFY:
                err = fcntl_dirnotify(fd, filp, argi);
                break;
        case F_SETPIPE_SZ:
        case F_GETPIPE_SZ:
                err = pipe_fcntl(filp, cmd, argi);
                break;
        case F_ADD_SEALS:
        case F_GET_SEALS:
                err = memfd_fcntl(filp, cmd, argi);
                break;
        case F_GET_RW_HINT:
                err = fcntl_get_rw_hint(filp, cmd, arg);
                break;
        case F_SET_RW_HINT:
                err = fcntl_set_rw_hint(filp, cmd, arg);
                break;
        default:
                break;
        }
        return err;
}

static int check_fcntl_cmd(unsigned cmd)
{
        switch (cmd) {
        case F_CREATED_QUERY:
        case F_DUPFD:
        case F_DUPFD_CLOEXEC:
        case F_DUPFD_QUERY:
        case F_GETFD:
        case F_SETFD:
        case F_GETFL:
                return 1;
        }
        return 0;
}

SYSCALL_DEFINE3(fcntl, unsigned int, fd, unsigned int, cmd, unsigned long, arg)
{        
        CLASS(fd_raw, f)(fd);
        long err;

        if (fd_empty(f))
                return -EBADF;

        if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) {
                if (!check_fcntl_cmd(cmd))
                        return -EBADF;
        }

        err = security_file_fcntl(fd_file(f), cmd, arg);
        if (!err)
                err = do_fcntl(fd, cmd, arg, fd_file(f));

        return err;
}

#if BITS_PER_LONG == 32
SYSCALL_DEFINE3(fcntl64, unsigned int, fd, unsigned int, cmd,
                unsigned long, arg)
{        
        void __user *argp = (void __user *)arg;
        CLASS(fd_raw, f)(fd);
        struct flock64 flock;
        long err;

        if (fd_empty(f))
                return -EBADF;

        if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) {
                if (!check_fcntl_cmd(cmd))
                        return -EBADF;
        }

        err = security_file_fcntl(fd_file(f), cmd, arg);
        if (err)
                return err;
        
        switch (cmd) {
        case F_GETLK64:
        case F_OFD_GETLK:
                err = -EFAULT;
                if (copy_from_user(&flock, argp, sizeof(flock)))
                        break;
                err = fcntl_getlk64(fd_file(f), cmd, &flock);
                if (!err && copy_to_user(argp, &flock, sizeof(flock)))
                        err = -EFAULT;
                break;
        case F_SETLK64:
        case F_SETLKW64:
        case F_OFD_SETLK:
        case F_OFD_SETLKW:
                err = -EFAULT;
                if (copy_from_user(&flock, argp, sizeof(flock)))
                        break;
                err = fcntl_setlk64(fd, fd_file(f), cmd, &flock);
                break;
        default:
                err = do_fcntl(fd, cmd, arg, fd_file(f));
                break;
        }
        return err;
}
#endif

#ifdef CONFIG_COMPAT
/* careful - don't use anywhere else */
#define copy_flock_fields(dst, src)                \
        (dst)->l_type = (src)->l_type;                \
        (dst)->l_whence = (src)->l_whence;        \
        (dst)->l_start = (src)->l_start;        \
        (dst)->l_len = (src)->l_len;                \
        (dst)->l_pid = (src)->l_pid;

static int get_compat_flock(struct flock *kfl, const struct compat_flock __user *ufl)
{
        struct compat_flock fl;

        if (copy_from_user(&fl, ufl, sizeof(struct compat_flock)))
                return -EFAULT;
        copy_flock_fields(kfl, &fl);
        return 0;
}

static int get_compat_flock64(struct flock *kfl, const struct compat_flock64 __user *ufl)
{
        struct compat_flock64 fl;

        if (copy_from_user(&fl, ufl, sizeof(struct compat_flock64)))
                return -EFAULT;
        copy_flock_fields(kfl, &fl);
        return 0;
}

static int put_compat_flock(const struct flock *kfl, struct compat_flock __user *ufl)
{
        struct compat_flock fl;

        memset(&fl, 0, sizeof(struct compat_flock));
        copy_flock_fields(&fl, kfl);
        if (copy_to_user(ufl, &fl, sizeof(struct compat_flock)))
                return -EFAULT;
        return 0;
}

static int put_compat_flock64(const struct flock *kfl, struct compat_flock64 __user *ufl)
{
        struct compat_flock64 fl;

        BUILD_BUG_ON(sizeof(kfl->l_start) > sizeof(ufl->l_start));
        BUILD_BUG_ON(sizeof(kfl->l_len) > sizeof(ufl->l_len));

        memset(&fl, 0, sizeof(struct compat_flock64));
        copy_flock_fields(&fl, kfl);
        if (copy_to_user(ufl, &fl, sizeof(struct compat_flock64)))
                return -EFAULT;
        return 0;
}
#undef copy_flock_fields

static unsigned int
convert_fcntl_cmd(unsigned int cmd)
{
        switch (cmd) {
        case F_GETLK64:
                return F_GETLK;
        case F_SETLK64:
                return F_SETLK;
        case F_SETLKW64:
                return F_SETLKW;
        }

        return cmd;
}

/*
 * GETLK was successful and we need to return the data, but it needs to fit in
 * the compat structure.
 * l_start shouldn't be too big, unless the original start + end is greater than
 * COMPAT_OFF_T_MAX, in which case the app was asking for trouble, so we return
 * -EOVERFLOW in that case.  l_len could be too big, in which case we just
 * truncate it, and only allow the app to see that part of the conflicting lock
 * that might make sense to it anyway
 */
static int fixup_compat_flock(struct flock *flock)
{
        if (flock->l_start > COMPAT_OFF_T_MAX)
                return -EOVERFLOW;
        if (flock->l_len > COMPAT_OFF_T_MAX)
                flock->l_len = COMPAT_OFF_T_MAX;
        return 0;
}

static long do_compat_fcntl64(unsigned int fd, unsigned int cmd,
                             compat_ulong_t arg)
{
        CLASS(fd_raw, f)(fd);
        struct flock flock;
        long err;

        if (fd_empty(f))
                return -EBADF;

        if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) {
                if (!check_fcntl_cmd(cmd))
                        return -EBADF;
        }

        err = security_file_fcntl(fd_file(f), cmd, arg);
        if (err)
                return err;

        switch (cmd) {
        case F_GETLK:
                err = get_compat_flock(&flock, compat_ptr(arg));
                if (err)
                        break;
                err = fcntl_getlk(fd_file(f), convert_fcntl_cmd(cmd), &flock);
                if (err)
                        break;
                err = fixup_compat_flock(&flock);
                if (!err)
                        err = put_compat_flock(&flock, compat_ptr(arg));
                break;
        case F_GETLK64:
        case F_OFD_GETLK:
                err = get_compat_flock64(&flock, compat_ptr(arg));
                if (err)
                        break;
                err = fcntl_getlk(fd_file(f), convert_fcntl_cmd(cmd), &flock);
                if (!err)
                        err = put_compat_flock64(&flock, compat_ptr(arg));
                break;
        case F_SETLK:
        case F_SETLKW:
                err = get_compat_flock(&flock, compat_ptr(arg));
                if (err)
                        break;
                err = fcntl_setlk(fd, fd_file(f), convert_fcntl_cmd(cmd), &flock);
                break;
        case F_SETLK64:
        case F_SETLKW64:
        case F_OFD_SETLK:
        case F_OFD_SETLKW:
                err = get_compat_flock64(&flock, compat_ptr(arg));
                if (err)
                        break;
                err = fcntl_setlk(fd, fd_file(f), convert_fcntl_cmd(cmd), &flock);
                break;
        default:
                err = do_fcntl(fd, cmd, arg, fd_file(f));
                break;
        }
        return err;
}

COMPAT_SYSCALL_DEFINE3(fcntl64, unsigned int, fd, unsigned int, cmd,
                       compat_ulong_t, arg)
{
        return do_compat_fcntl64(fd, cmd, arg);
}

COMPAT_SYSCALL_DEFINE3(fcntl, unsigned int, fd, unsigned int, cmd,
                       compat_ulong_t, arg)
{
        switch (cmd) {
        case F_GETLK64:
        case F_SETLK64:
        case F_SETLKW64:
        case F_OFD_GETLK:
        case F_OFD_SETLK:
        case F_OFD_SETLKW:
                return -EINVAL;
        }
        return do_compat_fcntl64(fd, cmd, arg);
}
#endif

/* Table to convert sigio signal codes into poll band bitmaps */

static const __poll_t band_table[NSIGPOLL] = {
        EPOLLIN | EPOLLRDNORM,                        /* POLL_IN */
        EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND,        /* POLL_OUT */
        EPOLLIN | EPOLLRDNORM | EPOLLMSG,                /* POLL_MSG */
        EPOLLERR,                                /* POLL_ERR */
        EPOLLPRI | EPOLLRDBAND,                        /* POLL_PRI */
        EPOLLHUP | EPOLLERR                        /* POLL_HUP */
};

static inline int sigio_perm(struct task_struct *p,
                             struct fown_struct *fown, int sig)
{
        const struct cred *cred;
        int ret;

        rcu_read_lock();
        cred = __task_cred(p);
        ret = ((uid_eq(fown->euid, GLOBAL_ROOT_UID) ||
                uid_eq(fown->euid, cred->suid) || uid_eq(fown->euid, cred->uid) ||
                uid_eq(fown->uid,  cred->suid) || uid_eq(fown->uid,  cred->uid)) &&
               !security_file_send_sigiotask(p, fown, sig));
        rcu_read_unlock();
        return ret;
}

static void send_sigio_to_task(struct task_struct *p,
                               struct fown_struct *fown,
                               int fd, int reason, enum pid_type type)
{
        /*
         * F_SETSIG can change ->signum lockless in parallel, make
         * sure we read it once and use the same value throughout.
         */
        int signum = READ_ONCE(fown->signum);

        if (!sigio_perm(p, fown, signum))
                return;

        switch (signum) {
                default: {
                        kernel_siginfo_t si;

                        /* Queue a rt signal with the appropriate fd as its
                           value.  We use SI_SIGIO as the source, not 
                           SI_KERNEL, since kernel signals always get 
                           delivered even if we can't queue.  Failure to
                           queue in this case _should_ be reported; we fall
                           back to SIGIO in that case. --sct */
                        clear_siginfo(&si);
                        si.si_signo = signum;
                        si.si_errno = 0;
                        si.si_code  = reason;
                        /*
                         * Posix definies POLL_IN and friends to be signal
                         * specific si_codes for SIG_POLL.  Linux extended
                         * these si_codes to other signals in a way that is
                         * ambiguous if other signals also have signal
                         * specific si_codes.  In that case use SI_SIGIO instead
                         * to remove the ambiguity.
                         */
                        if ((signum != SIGPOLL) && sig_specific_sicodes(signum))
                                si.si_code = SI_SIGIO;

                        /* Make sure we are called with one of the POLL_*
                           reasons, otherwise we could leak kernel stack into
                           userspace.  */
                        BUG_ON((reason < POLL_IN) || ((reason - POLL_IN) >= NSIGPOLL));
                        if (reason - POLL_IN >= NSIGPOLL)
                                si.si_band  = ~0L;
                        else
                                si.si_band = mangle_poll(band_table[reason - POLL_IN]);
                        si.si_fd    = fd;
                        if (!do_send_sig_info(signum, &si, p, type))
                                break;
                }
                        fallthrough;        /* fall back on the old plain SIGIO signal */
                case 0:
                        do_send_sig_info(SIGIO, SEND_SIG_PRIV, p, type);
        }
}

void send_sigio(struct fown_struct *fown, int fd, int band)
{
        struct task_struct *p;
        enum pid_type type;
        unsigned long flags;
        struct pid *pid;
        
        read_lock_irqsave(&fown->lock, flags);

        type = fown->pid_type;
        pid = fown->pid;
        if (!pid)
                goto out_unlock_fown;

        if (type <= PIDTYPE_TGID) {
                rcu_read_lock();
                p = pid_task(pid, PIDTYPE_PID);
                if (p)
                        send_sigio_to_task(p, fown, fd, band, type);
                rcu_read_unlock();
        } else {
                read_lock(&tasklist_lock);
                do_each_pid_task(pid, type, p) {
                        send_sigio_to_task(p, fown, fd, band, type);
                } while_each_pid_task(pid, type, p);
                read_unlock(&tasklist_lock);
        }
 out_unlock_fown:
        read_unlock_irqrestore(&fown->lock, flags);
}

static void send_sigurg_to_task(struct task_struct *p,
                                struct fown_struct *fown, enum pid_type type)
{
        if (sigio_perm(p, fown, SIGURG))
                do_send_sig_info(SIGURG, SEND_SIG_PRIV, p, type);
}

int send_sigurg(struct file *file)
{
        struct fown_struct *fown;
        struct task_struct *p;
        enum pid_type type;
        struct pid *pid;
        unsigned long flags;
        int ret = 0;
        
        fown = file_f_owner(file);
        if (!fown)
                return 0;

        read_lock_irqsave(&fown->lock, flags);

        type = fown->pid_type;
        pid = fown->pid;
        if (!pid)
                goto out_unlock_fown;

        ret = 1;

        if (type <= PIDTYPE_TGID) {
                rcu_read_lock();
                p = pid_task(pid, PIDTYPE_PID);
                if (p)
                        send_sigurg_to_task(p, fown, type);
                rcu_read_unlock();
        } else {
                read_lock(&tasklist_lock);
                do_each_pid_task(pid, type, p) {
                        send_sigurg_to_task(p, fown, type);
                } while_each_pid_task(pid, type, p);
                read_unlock(&tasklist_lock);
        }
 out_unlock_fown:
        read_unlock_irqrestore(&fown->lock, flags);
        return ret;
}

static DEFINE_SPINLOCK(fasync_lock);
static struct kmem_cache *fasync_cache __ro_after_init;

/*
 * Remove a fasync entry. If successfully removed, return
 * positive and clear the FASYNC flag. If no entry exists,
 * do nothing and return 0.
 *
 * NOTE! It is very important that the FASYNC flag always
 * match the state "is the filp on a fasync list".
 *
 */
int fasync_remove_entry(struct file *filp, struct fasync_struct **fapp)
{
        struct fasync_struct *fa, **fp;
        int result = 0;

        spin_lock(&filp->f_lock);
        spin_lock(&fasync_lock);
        for (fp = fapp; (fa = *fp) != NULL; fp = &fa->fa_next) {
                if (fa->fa_file != filp)
                        continue;

                write_lock_irq(&fa->fa_lock);
                fa->fa_file = NULL;
                write_unlock_irq(&fa->fa_lock);

                *fp = fa->fa_next;
                kfree_rcu(fa, fa_rcu);
                filp->f_flags &= ~FASYNC;
                result = 1;
                break;
        }
        spin_unlock(&fasync_lock);
        spin_unlock(&filp->f_lock);
        return result;
}

struct fasync_struct *fasync_alloc(void)
{
        return kmem_cache_alloc(fasync_cache, GFP_KERNEL);
}

/*
 * NOTE! This can be used only for unused fasync entries:
 * entries that actually got inserted on the fasync list
 * need to be released by rcu - see fasync_remove_entry.
 */
void fasync_free(struct fasync_struct *new)
{
        kmem_cache_free(fasync_cache, new);
}

/*
 * Insert a new entry into the fasync list.  Return the pointer to the
 * old one if we didn't use the new one.
 *
 * NOTE! It is very important that the FASYNC flag always
 * match the state "is the filp on a fasync list".
 */
struct fasync_struct *fasync_insert_entry(int fd, struct file *filp, struct fasync_struct **fapp, struct fasync_struct *new)
{
        struct fasync_struct *fa, **fp;

        spin_lock(&filp->f_lock);
        spin_lock(&fasync_lock);
        for (fp = fapp; (fa = *fp) != NULL; fp = &fa->fa_next) {
                if (fa->fa_file != filp)
                        continue;

                write_lock_irq(&fa->fa_lock);
                fa->fa_fd = fd;
                write_unlock_irq(&fa->fa_lock);
                goto out;
        }

        rwlock_init(&new->fa_lock);
        new->magic = FASYNC_MAGIC;
        new->fa_file = filp;
        new->fa_fd = fd;
        new->fa_next = *fapp;
        rcu_assign_pointer(*fapp, new);
        filp->f_flags |= FASYNC;

out:
        spin_unlock(&fasync_lock);
        spin_unlock(&filp->f_lock);
        return fa;
}

/*
 * Add a fasync entry. Return negative on error, positive if
 * added, and zero if did nothing but change an existing one.
 */
static int fasync_add_entry(int fd, struct file *filp, struct fasync_struct **fapp)
{
        struct fasync_struct *new;

        new = fasync_alloc();
        if (!new)
                return -ENOMEM;

        /*
         * fasync_insert_entry() returns the old (update) entry if
         * it existed.
         *
         * So free the (unused) new entry and return 0 to let the
         * caller know that we didn't add any new fasync entries.
         */
        if (fasync_insert_entry(fd, filp, fapp, new)) {
                fasync_free(new);
                return 0;
        }

        return 1;
}

/*
 * fasync_helper() is used by almost all character device drivers
 * to set up the fasync queue, and for regular files by the file
 * lease code. It returns negative on error, 0 if it did no changes
 * and positive if it added/deleted the entry.
 */
int fasync_helper(int fd, struct file * filp, int on, struct fasync_struct **fapp)
{
        if (!on)
                return fasync_remove_entry(filp, fapp);
        return fasync_add_entry(fd, filp, fapp);
}

EXPORT_SYMBOL(fasync_helper);

/*
 * rcu_read_lock() is held
 */
static void kill_fasync_rcu(struct fasync_struct *fa, int sig, int band)
{
        while (fa) {
                struct fown_struct *fown;
                unsigned long flags;

                if (fa->magic != FASYNC_MAGIC) {
                        printk(KERN_ERR "kill_fasync: bad magic number in "
                               "fasync_struct!\n");
                        return;
                }
                read_lock_irqsave(&fa->fa_lock, flags);
                if (fa->fa_file) {
                        fown = file_f_owner(fa->fa_file);
                        if (!fown)
                                goto next;
                        /* Don't send SIGURG to processes which have not set a
                           queued signum: SIGURG has its own default signalling
                           mechanism. */
                        if (!(sig == SIGURG && fown->signum == 0))
                                send_sigio(fown, fa->fa_fd, band);
                }
next:
                read_unlock_irqrestore(&fa->fa_lock, flags);
                fa = rcu_dereference(fa->fa_next);
        }
}

void kill_fasync(struct fasync_struct **fp, int sig, int band)
{
        /* First a quick test without locking: usually
         * the list is empty.
         */
        if (*fp) {
                rcu_read_lock();
                kill_fasync_rcu(rcu_dereference(*fp), sig, band);
                rcu_read_unlock();
        }
}
EXPORT_SYMBOL(kill_fasync);

static int __init fcntl_init(void)
{
        /*
         * Please add new bits here to ensure allocation uniqueness.
         * Exceptions: O_NONBLOCK is a two bit define on parisc; O_NDELAY
         * is defined as O_NONBLOCK on some platforms and not on others.
         */
        BUILD_BUG_ON(20 - 1 /* for O_RDONLY being 0 */ !=
                HWEIGHT32(
                        (VALID_OPEN_FLAGS & ~(O_NONBLOCK | O_NDELAY)) |
                        __FMODE_EXEC));

        fasync_cache = kmem_cache_create("fasync_cache",
                                         sizeof(struct fasync_struct), 0,
                                         SLAB_PANIC | SLAB_ACCOUNT, NULL);
        return 0;
}

module_init(fcntl_init)












































    7 












    7 






















    7 













    7 




























    7 




    7 













    7 





















    7 


    7 


    7 












































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
// SPDX-License-Identifier: GPL-2.0 or MIT
/*
 * Copyright 2018 Noralf Trønnes
 */

#include <linux/export.h>
#include <linux/iosys-map.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/seq_file.h>
#include <linux/slab.h>

#include <drm/drm_client.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_fourcc.h>
#include <drm/drm_framebuffer.h>
#include <drm/drm_gem.h>
#include <drm/drm_mode.h>
#include <drm/drm_print.h>

#include "drm_crtc_internal.h"
#include "drm_internal.h"

/**
 * DOC: overview
 *
 * This library provides support for clients running in the kernel like fbdev and bootsplash.
 *
 * GEM drivers which provide a GEM based dumb buffer with a virtual address are supported.
 */

static int drm_client_open(struct drm_client_dev *client)
{
        struct drm_device *dev = client->dev;
        struct drm_file *file;

        file = drm_file_alloc(dev->primary);
        if (IS_ERR(file))
                return PTR_ERR(file);

        mutex_lock(&dev->filelist_mutex);
        list_add(&file->lhead, &dev->filelist_internal);
        mutex_unlock(&dev->filelist_mutex);

        client->file = file;

        return 0;
}

static void drm_client_close(struct drm_client_dev *client)
{
        struct drm_device *dev = client->dev;

        mutex_lock(&dev->filelist_mutex);
        list_del(&client->file->lhead);
        mutex_unlock(&dev->filelist_mutex);

        drm_file_free(client->file);
}

/**
 * drm_client_init - Initialise a DRM client
 * @dev: DRM device
 * @client: DRM client
 * @name: Client name
 * @funcs: DRM client functions (optional)
 *
 * This initialises the client and opens a &drm_file.
 * Use drm_client_register() to complete the process.
 * The caller needs to hold a reference on @dev before calling this function.
 * The client is freed when the &drm_device is unregistered. See drm_client_release().
 *
 * Returns:
 * Zero on success or negative error code on failure.
 */
int drm_client_init(struct drm_device *dev, struct drm_client_dev *client,
                    const char *name, const struct drm_client_funcs *funcs)
{
        int ret;

        if (!drm_core_check_feature(dev, DRIVER_MODESET) || !dev->driver->dumb_create)
                return -EOPNOTSUPP;

        client->dev = dev;
        client->name = name;
        client->funcs = funcs;

        ret = drm_client_modeset_create(client);
        if (ret)
                return ret;

        ret = drm_client_open(client);
        if (ret)
                goto err_free;

        drm_dev_get(dev);

        return 0;

err_free:
        drm_client_modeset_free(client);
        return ret;
}
EXPORT_SYMBOL(drm_client_init);

/**
 * drm_client_register - Register client
 * @client: DRM client
 *
 * Add the client to the &drm_device client list to activate its callbacks.
 * @client must be initialized by a call to drm_client_init(). After
 * drm_client_register() it is no longer permissible to call drm_client_release()
 * directly (outside the unregister callback), instead cleanup will happen
 * automatically on driver unload.
 *
 * Registering a client generates a hotplug event that allows the client
 * to set up its display from pre-existing outputs. The client must have
 * initialized its state to able to handle the hotplug event successfully.
 */
void drm_client_register(struct drm_client_dev *client)
{
        struct drm_device *dev = client->dev;
        int ret;

        mutex_lock(&dev->clientlist_mutex);
        list_add(&client->list, &dev->clientlist);

        if (client->funcs && client->funcs->hotplug) {
                /*
                 * Perform an initial hotplug event to pick up the
                 * display configuration for the client. This step
                 * has to be performed *after* registering the client
                 * in the list of clients, or a concurrent hotplug
                 * event might be lost; leaving the display off.
                 *
                 * Hold the clientlist_mutex as for a regular hotplug
                 * event.
                 */
                ret = client->funcs->hotplug(client);
                if (ret)
                        drm_dbg_kms(dev, "client hotplug ret=%d\n", ret);
        }
        mutex_unlock(&dev->clientlist_mutex);
}
EXPORT_SYMBOL(drm_client_register);

/**
 * drm_client_release - Release DRM client resources
 * @client: DRM client
 *
 * Releases resources by closing the &drm_file that was opened by drm_client_init().
 * It is called automatically if the &drm_client_funcs.unregister callback is _not_ set.
 *
 * This function should only be called from the unregister callback. An exception
 * is fbdev which cannot free the buffer if userspace has open file descriptors.
 *
 * Note:
 * Clients cannot initiate a release by themselves. This is done to keep the code simple.
 * The driver has to be unloaded before the client can be unloaded.
 */
void drm_client_release(struct drm_client_dev *client)
{
        struct drm_device *dev = client->dev;

        drm_dbg_kms(dev, "%s\n", client->name);

        drm_client_modeset_free(client);
        drm_client_close(client);
        drm_dev_put(dev);
}
EXPORT_SYMBOL(drm_client_release);

static void drm_client_buffer_delete(struct drm_client_buffer *buffer)
{
        if (buffer->gem) {
                drm_gem_vunmap(buffer->gem, &buffer->map);
                drm_gem_object_put(buffer->gem);
        }

        kfree(buffer);
}

static struct drm_client_buffer *
drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height,
                         u32 format, u32 *handle)
{
        const struct drm_format_info *info = drm_format_info(format);
        struct drm_mode_create_dumb dumb_args = { };
        struct drm_device *dev = client->dev;
        struct drm_client_buffer *buffer;
        struct drm_gem_object *obj;
        int ret;

        buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
        if (!buffer)
                return ERR_PTR(-ENOMEM);

        buffer->client = client;

        dumb_args.width = width;
        dumb_args.height = height;
        dumb_args.bpp = drm_format_info_bpp(info, 0);
        ret = drm_mode_create_dumb(dev, &dumb_args, client->file);
        if (ret)
                goto err_delete;

        obj = drm_gem_object_lookup(client->file, dumb_args.handle);
        if (!obj)  {
                ret = -ENOENT;
                goto err_delete;
        }

        buffer->pitch = dumb_args.pitch;
        buffer->gem = obj;
        *handle = dumb_args.handle;

        return buffer;

err_delete:
        drm_client_buffer_delete(buffer);

        return ERR_PTR(ret);
}

/**
 * drm_client_buffer_vmap_local - Map DRM client buffer into address space
 * @buffer: DRM client buffer
 * @map_copy: Returns the mapped memory's address
 *
 * This function maps a client buffer into kernel address space. If the
 * buffer is already mapped, it returns the existing mapping's address.
 *
 * Client buffer mappings are not ref'counted. Each call to
 * drm_client_buffer_vmap_local() should be closely followed by a call to
 * drm_client_buffer_vunmap_local(). See drm_client_buffer_vmap() for
 * long-term mappings.
 *
 * The returned address is a copy of the internal value. In contrast to
 * other vmap interfaces, you don't need it for the client's vunmap
 * function. So you can modify it at will during blit and draw operations.
 *
 * Returns:
 *        0 on success, or a negative errno code otherwise.
 */
int drm_client_buffer_vmap_local(struct drm_client_buffer *buffer,
                                 struct iosys_map *map_copy)
{
        struct drm_gem_object *gem = buffer->gem;
        struct iosys_map *map = &buffer->map;
        int ret;

        drm_gem_lock(gem);

        ret = drm_gem_vmap_locked(gem, map);
        if (ret)
                goto err_drm_gem_vmap_unlocked;
        *map_copy = *map;

        return 0;

err_drm_gem_vmap_unlocked:
        drm_gem_unlock(gem);
        return ret;
}
EXPORT_SYMBOL(drm_client_buffer_vmap_local);

/**
 * drm_client_buffer_vunmap_local - Unmap DRM client buffer
 * @buffer: DRM client buffer
 *
 * This function removes a client buffer's memory mapping established
 * with drm_client_buffer_vunmap_local(). Calling this function is only
 * required by clients that manage their buffer mappings by themselves.
 */
void drm_client_buffer_vunmap_local(struct drm_client_buffer *buffer)
{
        struct drm_gem_object *gem = buffer->gem;
        struct iosys_map *map = &buffer->map;

        drm_gem_vunmap_locked(gem, map);
        drm_gem_unlock(gem);
}
EXPORT_SYMBOL(drm_client_buffer_vunmap_local);

/**
 * drm_client_buffer_vmap - Map DRM client buffer into address space
 * @buffer: DRM client buffer
 * @map_copy: Returns the mapped memory's address
 *
 * This function maps a client buffer into kernel address space. If the
 * buffer is already mapped, it returns the existing mapping's address.
 *
 * Client buffer mappings are not ref'counted. Each call to
 * drm_client_buffer_vmap() should be followed by a call to
 * drm_client_buffer_vunmap(); or the client buffer should be mapped
 * throughout its lifetime.
 *
 * The returned address is a copy of the internal value. In contrast to
 * other vmap interfaces, you don't need it for the client's vunmap
 * function. So you can modify it at will during blit and draw operations.
 *
 * Returns:
 *        0 on success, or a negative errno code otherwise.
 */
int drm_client_buffer_vmap(struct drm_client_buffer *buffer,
                           struct iosys_map *map_copy)
{
        int ret;

        ret = drm_gem_vmap(buffer->gem, &buffer->map);
        if (ret)
                return ret;
        *map_copy = buffer->map;

        return 0;
}
EXPORT_SYMBOL(drm_client_buffer_vmap);

/**
 * drm_client_buffer_vunmap - Unmap DRM client buffer
 * @buffer: DRM client buffer
 *
 * This function removes a client buffer's memory mapping. Calling this
 * function is only required by clients that manage their buffer mappings
 * by themselves.
 */
void drm_client_buffer_vunmap(struct drm_client_buffer *buffer)
{
        drm_gem_vunmap(buffer->gem, &buffer->map);
}
EXPORT_SYMBOL(drm_client_buffer_vunmap);

static void drm_client_buffer_rmfb(struct drm_client_buffer *buffer)
{
        int ret;

        if (!buffer->fb)
                return;

        ret = drm_mode_rmfb(buffer->client->dev, buffer->fb->base.id, buffer->client->file);
        if (ret)
                drm_err(buffer->client->dev,
                        "Error removing FB:%u (%d)\n", buffer->fb->base.id, ret);

        buffer->fb = NULL;
}

static int drm_client_buffer_addfb(struct drm_client_buffer *buffer,
                                   u32 width, u32 height, u32 format,
                                   u32 handle)
{
        struct drm_client_dev *client = buffer->client;
        struct drm_mode_fb_cmd2 fb_req = { };
        int ret;

        fb_req.width = width;
        fb_req.height = height;
        fb_req.pixel_format = format;
        fb_req.handles[0] = handle;
        fb_req.pitches[0] = buffer->pitch;

        ret = drm_mode_addfb2(client->dev, &fb_req, client->file);
        if (ret)
                return ret;

        buffer->fb = drm_framebuffer_lookup(client->dev, buffer->client->file, fb_req.fb_id);
        if (WARN_ON(!buffer->fb))
                return -ENOENT;

        /* drop the reference we picked up in framebuffer lookup */
        drm_framebuffer_put(buffer->fb);

        strscpy(buffer->fb->comm, client->name, TASK_COMM_LEN);

        return 0;
}

/**
 * drm_client_framebuffer_create - Create a client framebuffer
 * @client: DRM client
 * @width: Framebuffer width
 * @height: Framebuffer height
 * @format: Buffer format
 *
 * This function creates a &drm_client_buffer which consists of a
 * &drm_framebuffer backed by a dumb buffer.
 * Call drm_client_framebuffer_delete() to free the buffer.
 *
 * Returns:
 * Pointer to a client buffer or an error pointer on failure.
 */
struct drm_client_buffer *
drm_client_framebuffer_create(struct drm_client_dev *client, u32 width, u32 height, u32 format)
{
        struct drm_client_buffer *buffer;
        u32 handle;
        int ret;

        buffer = drm_client_buffer_create(client, width, height, format,
                                          &handle);
        if (IS_ERR(buffer))
                return buffer;

        ret = drm_client_buffer_addfb(buffer, width, height, format, handle);

        /*
         * The handle is only needed for creating the framebuffer, destroy it
         * again to solve a circular dependency should anybody export the GEM
         * object as DMA-buf. The framebuffer and our buffer structure are still
         * holding references to the GEM object to prevent its destruction.
         */
        drm_mode_destroy_dumb(client->dev, handle, client->file);

        if (ret) {
                drm_client_buffer_delete(buffer);
                return ERR_PTR(ret);
        }

        return buffer;
}
EXPORT_SYMBOL(drm_client_framebuffer_create);

/**
 * drm_client_framebuffer_delete - Delete a client framebuffer
 * @buffer: DRM client buffer (can be NULL)
 */
void drm_client_framebuffer_delete(struct drm_client_buffer *buffer)
{
        if (!buffer)
                return;

        drm_client_buffer_rmfb(buffer);
        drm_client_buffer_delete(buffer);
}
EXPORT_SYMBOL(drm_client_framebuffer_delete);

/**
 * drm_client_framebuffer_flush - Manually flush client framebuffer
 * @buffer: DRM client buffer (can be NULL)
 * @rect: Damage rectangle (if NULL flushes all)
 *
 * This calls &drm_framebuffer_funcs->dirty (if present) to flush buffer changes
 * for drivers that need it.
 *
 * Returns:
 * Zero on success or negative error code on failure.
 */
int drm_client_framebuffer_flush(struct drm_client_buffer *buffer, struct drm_rect *rect)
{
        if (!buffer || !buffer->fb || !buffer->fb->funcs->dirty)
                return 0;

        if (rect) {
                struct drm_clip_rect clip = {
                        .x1 = rect->x1,
                        .y1 = rect->y1,
                        .x2 = rect->x2,
                        .y2 = rect->y2,
                };

                return buffer->fb->funcs->dirty(buffer->fb, buffer->client->file,
                                                0, 0, &clip, 1);
        }

        return buffer->fb->funcs->dirty(buffer->fb, buffer->client->file,
                                        0, 0, NULL, 0);
}
EXPORT_SYMBOL(drm_client_framebuffer_flush);



























































   14 











    2 


    3 



    7 






    2 














    4 













    4 


















    2 





    2 


















    2 











    1 








    1 






    1 





















    2 


    2 




















    4 






    3 










    1 




    1 







    7 




















    7 




    9 










   11 






   11 






    3 

















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
/*
 * Resizable simple ram filesystem for Linux.
 *
 * Copyright (C) 2000 Linus Torvalds.
 *               2000 Transmeta Corp.
 *
 * Usage limits added by David Gibson, Linuxcare Australia.
 * This file is released under the GPL.
 */

/*
 * NOTE! This filesystem is probably most useful
 * not as a real filesystem, but as an example of
 * how virtual filesystems can be written.
 *
 * It doesn't get much simpler than this. Consider
 * that this file implements the full semantics of
 * a POSIX-compliant read-write filesystem.
 *
 * Note in particular how the filesystem does not
 * need to implement any data structures of its own
 * to keep track of the virtual data: using the VFS
 * caches is sufficient.
 */

#include <linux/fs.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/time.h>
#include <linux/init.h>
#include <linux/string.h>
#include <linux/backing-dev.h>
#include <linux/ramfs.h>
#include <linux/sched.h>
#include <linux/parser.h>
#include <linux/magic.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/fs_context.h>
#include <linux/fs_parser.h>
#include <linux/seq_file.h>
#include "internal.h"

struct ramfs_mount_opts {
        umode_t mode;
};

struct ramfs_fs_info {
        struct ramfs_mount_opts mount_opts;
};

#define RAMFS_DEFAULT_MODE        0755

static const struct super_operations ramfs_ops;
static const struct inode_operations ramfs_dir_inode_operations;

struct inode *ramfs_get_inode(struct super_block *sb,
                                const struct inode *dir, umode_t mode, dev_t dev)
{
        struct inode * inode = new_inode(sb);

        if (inode) {
                inode->i_ino = get_next_ino();
                inode_init_owner(&nop_mnt_idmap, inode, dir, mode);
                inode->i_mapping->a_ops = &ram_aops;
                mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
                mapping_set_unevictable(inode->i_mapping);
                simple_inode_init_ts(inode);
                switch (mode & S_IFMT) {
                default:
                        init_special_inode(inode, mode, dev);
                        break;
                case S_IFREG:
                        inode->i_op = &ramfs_file_inode_operations;
                        inode->i_fop = &ramfs_file_operations;
                        break;
                case S_IFDIR:
                        inode->i_op = &ramfs_dir_inode_operations;
                        inode->i_fop = &simple_dir_operations;

                        /* directory inodes start off with i_nlink == 2 (for "." entry) */
                        inc_nlink(inode);
                        break;
                case S_IFLNK:
                        inode->i_op = &page_symlink_inode_operations;
                        inode_nohighmem(inode);
                        break;
                }
        }
        return inode;
}

/*
 * File creation. Allocate an inode, and we're done..
 */
/* SMP-safe */
static int
ramfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
            struct dentry *dentry, umode_t mode, dev_t dev)
{
        struct inode * inode = ramfs_get_inode(dir->i_sb, dir, mode, dev);
        int error = -ENOSPC;

        if (inode) {
                error = security_inode_init_security(inode, dir,
                                                     &dentry->d_name, NULL,
                                                     NULL);
                if (error) {
                        iput(inode);
                        goto out;
                }

                d_instantiate(dentry, inode);
                dget(dentry);        /* Extra count - pin the dentry in core */
                error = 0;
                inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir));
        }
out:
        return error;
}

static struct dentry *ramfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
                                 struct dentry *dentry, umode_t mode)
{
        int retval = ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFDIR, 0);
        if (!retval)
                inc_nlink(dir);
        return ERR_PTR(retval);
}

static int ramfs_create(struct mnt_idmap *idmap, struct inode *dir,
                        struct dentry *dentry, umode_t mode, bool excl)
{
        return ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0);
}

static int ramfs_symlink(struct mnt_idmap *idmap, struct inode *dir,
                         struct dentry *dentry, const char *symname)
{
        struct inode *inode;
        int error = -ENOSPC;

        inode = ramfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
        if (inode) {
                int l = strlen(symname)+1;

                error = security_inode_init_security(inode, dir,
                                                     &dentry->d_name, NULL,
                                                     NULL);
                if (error) {
                        iput(inode);
                        goto out;
                }

                error = page_symlink(inode, symname, l);
                if (!error) {
                        d_instantiate(dentry, inode);
                        dget(dentry);
                        inode_set_mtime_to_ts(dir,
                                              inode_set_ctime_current(dir));
                } else
                        iput(inode);
        }
out:
        return error;
}

static int ramfs_tmpfile(struct mnt_idmap *idmap,
                         struct inode *dir, struct file *file, umode_t mode)
{
        struct inode *inode;
        int error;

        inode = ramfs_get_inode(dir->i_sb, dir, mode, 0);
        if (!inode)
                return -ENOSPC;

        error = security_inode_init_security(inode, dir,
                                             &file_dentry(file)->d_name, NULL,
                                             NULL);
        if (error) {
                iput(inode);
                goto out;
        }

        d_tmpfile(file, inode);
out:
        return finish_open_simple(file, error);
}

static const struct inode_operations ramfs_dir_inode_operations = {
        .create                = ramfs_create,
        .lookup                = simple_lookup,
        .link                = simple_link,
        .unlink                = simple_unlink,
        .symlink        = ramfs_symlink,
        .mkdir                = ramfs_mkdir,
        .rmdir                = simple_rmdir,
        .mknod                = ramfs_mknod,
        .rename                = simple_rename,
        .tmpfile        = ramfs_tmpfile,
};

/*
 * Display the mount options in /proc/mounts.
 */
static int ramfs_show_options(struct seq_file *m, struct dentry *root)
{
        struct ramfs_fs_info *fsi = root->d_sb->s_fs_info;

        if (fsi->mount_opts.mode != RAMFS_DEFAULT_MODE)
                seq_printf(m, ",mode=%o", fsi->mount_opts.mode);
        return 0;
}

static const struct super_operations ramfs_ops = {
        .statfs                = simple_statfs,
        .drop_inode        = generic_delete_inode,
        .show_options        = ramfs_show_options,
};

enum ramfs_param {
        Opt_mode,
};

const struct fs_parameter_spec ramfs_fs_parameters[] = {
        fsparam_u32oct("mode",        Opt_mode),
        {}
};

static int ramfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
{
        struct fs_parse_result result;
        struct ramfs_fs_info *fsi = fc->s_fs_info;
        int opt;

        opt = fs_parse(fc, ramfs_fs_parameters, param, &result);
        if (opt == -ENOPARAM) {
                opt = vfs_parse_fs_param_source(fc, param);
                if (opt != -ENOPARAM)
                        return opt;
                /*
                 * We might like to report bad mount options here;
                 * but traditionally ramfs has ignored all mount options,
                 * and as it is used as a !CONFIG_SHMEM simple substitute
                 * for tmpfs, better continue to ignore other mount options.
                 */
                return 0;
        }
        if (opt < 0)
                return opt;

        switch (opt) {
        case Opt_mode:
                fsi->mount_opts.mode = result.uint_32 & S_IALLUGO;
                break;
        }

        return 0;
}

static int ramfs_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct ramfs_fs_info *fsi = sb->s_fs_info;
        struct inode *inode;

        sb->s_maxbytes                = MAX_LFS_FILESIZE;
        sb->s_blocksize                = PAGE_SIZE;
        sb->s_blocksize_bits        = PAGE_SHIFT;
        sb->s_magic                = RAMFS_MAGIC;
        sb->s_op                = &ramfs_ops;
        sb->s_d_flags                = DCACHE_DONTCACHE;
        sb->s_time_gran                = 1;

        inode = ramfs_get_inode(sb, NULL, S_IFDIR | fsi->mount_opts.mode, 0);
        sb->s_root = d_make_root(inode);
        if (!sb->s_root)
                return -ENOMEM;

        return 0;
}

static int ramfs_get_tree(struct fs_context *fc)
{
        return get_tree_nodev(fc, ramfs_fill_super);
}

static void ramfs_free_fc(struct fs_context *fc)
{
        kfree(fc->s_fs_info);
}

static const struct fs_context_operations ramfs_context_ops = {
        .free                = ramfs_free_fc,
        .parse_param        = ramfs_parse_param,
        .get_tree        = ramfs_get_tree,
};

int ramfs_init_fs_context(struct fs_context *fc)
{
        struct ramfs_fs_info *fsi;

        fsi = kzalloc(sizeof(*fsi), GFP_KERNEL);
        if (!fsi)
                return -ENOMEM;

        fsi->mount_opts.mode = RAMFS_DEFAULT_MODE;
        fc->s_fs_info = fsi;
        fc->ops = &ramfs_context_ops;
        return 0;
}

void ramfs_kill_sb(struct super_block *sb)
{
        kfree(sb->s_fs_info);
        kill_litter_super(sb);
}

static struct file_system_type ramfs_fs_type = {
        .name                = "ramfs",
        .init_fs_context = ramfs_init_fs_context,
        .parameters        = ramfs_fs_parameters,
        .kill_sb        = ramfs_kill_sb,
        .fs_flags        = FS_USERNS_MOUNT,
};

static int __init init_ramfs_fs(void)
{
        return register_filesystem(&ramfs_fs_type);
}
fs_initcall(init_ramfs_fs);

























































































































   12 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef BLKTRACE_H
#define BLKTRACE_H

#include <linux/blk-mq.h>
#include <linux/relay.h>
#include <linux/compat.h>
#include <uapi/linux/blktrace_api.h>
#include <linux/list.h>
#include <linux/blk_types.h>

#if defined(CONFIG_BLK_DEV_IO_TRACE)

#include <linux/sysfs.h>

struct blk_trace {
        int trace_state;
        struct rchan *rchan;
        unsigned long __percpu *sequence;
        unsigned char __percpu *msg_data;
        u16 act_mask;
        u64 start_lba;
        u64 end_lba;
        u32 pid;
        u32 dev;
        struct dentry *dir;
        struct list_head running_list;
        atomic_t dropped;
};

extern int blk_trace_ioctl(struct block_device *, unsigned, char __user *);
extern void blk_trace_shutdown(struct request_queue *);
__printf(3, 4) void __blk_trace_note_message(struct blk_trace *bt,
                struct cgroup_subsys_state *css, const char *fmt, ...);

/**
 * blk_add_trace_msg - Add a (simple) message to the blktrace stream
 * @q:                queue the io is for
 * @fmt:        format to print message in
 * args...        Variable argument list for format
 *
 * Description:
 *     Records a (simple) message onto the blktrace stream.
 *
 *     NOTE: BLK_TN_MAX_MSG characters are output at most.
 *     NOTE: Can not use 'static inline' due to presence of var args...
 *
 **/
#define blk_add_cgroup_trace_msg(q, css, fmt, ...)                        \
        do {                                                                \
                struct blk_trace *bt;                                        \
                                                                        \
                rcu_read_lock();                                        \
                bt = rcu_dereference((q)->blk_trace);                        \
                if (unlikely(bt))                                        \
                        __blk_trace_note_message(bt, css, fmt, ##__VA_ARGS__);\
                rcu_read_unlock();                                        \
        } while (0)
#define blk_add_trace_msg(q, fmt, ...)                                        \
        blk_add_cgroup_trace_msg(q, NULL, fmt, ##__VA_ARGS__)
#define BLK_TN_MAX_MSG                128

static inline bool blk_trace_note_message_enabled(struct request_queue *q)
{
        struct blk_trace *bt;
        bool ret;

        rcu_read_lock();
        bt = rcu_dereference(q->blk_trace);
        ret = bt && (bt->act_mask & BLK_TC_NOTIFY);
        rcu_read_unlock();
        return ret;
}

extern void blk_add_driver_data(struct request *rq, void *data, size_t len);
extern int blk_trace_setup(struct request_queue *q, char *name, dev_t dev,
                           struct block_device *bdev,
                           char __user *arg);
extern int blk_trace_startstop(struct request_queue *q, int start);
extern int blk_trace_remove(struct request_queue *q);

#else /* !CONFIG_BLK_DEV_IO_TRACE */
# define blk_trace_ioctl(bdev, cmd, arg)                (-ENOTTY)
# define blk_trace_shutdown(q)                                do { } while (0)
# define blk_add_driver_data(rq, data, len)                do {} while (0)
# define blk_trace_setup(q, name, dev, bdev, arg)        (-ENOTTY)
# define blk_trace_startstop(q, start)                        (-ENOTTY)
# define blk_add_trace_msg(q, fmt, ...)                        do { } while (0)
# define blk_add_cgroup_trace_msg(q, cg, fmt, ...)        do { } while (0)
# define blk_trace_note_message_enabled(q)                (false)

static inline int blk_trace_remove(struct request_queue *q)
{
        return -ENOTTY;
}
#endif /* CONFIG_BLK_DEV_IO_TRACE */

#ifdef CONFIG_COMPAT

struct compat_blk_user_trace_setup {
        char name[BLKTRACE_BDEV_SIZE];
        u16 act_mask;
        u32 buf_size;
        u32 buf_nr;
        compat_u64 start_lba;
        compat_u64 end_lba;
        u32 pid;
};
#define BLKTRACESETUP32 _IOWR(0x12, 115, struct compat_blk_user_trace_setup)

#endif

void blk_fill_rwbs(char *rwbs, blk_opf_t opf);

static inline sector_t blk_rq_trace_sector(struct request *rq)
{
        /*
         * Tracing should ignore starting sector for passthrough requests and
         * requests where starting sector didn't get set.
         */
        if (blk_rq_is_passthrough(rq) || blk_rq_pos(rq) == (sector_t)-1)
                return 0;
        return blk_rq_pos(rq);
}

static inline unsigned int blk_rq_trace_nr_sectors(struct request *rq)
{
        return blk_rq_is_passthrough(rq) ? 0 : blk_rq_sectors(rq);
}

#endif









































   98 
  201 

    2 



















   23 










    3 











    1 
   34 



   63 



   63 
   63 

   79 










    2 




   82 

   84 
   83 





   79 


   22 





   17 

    9 

   57 





   22 
   73 
   60 


   20 



   26 
   58 

   58 




   65 


























  177 





  142 
  178 





   79 

   84 

   79 


   22 
  177 




 4185 


 4194 
 4184 
 4176 
  140 
 4194 













  640 
  638 







   70 


   70 
   70 

    9 
   65 

   65 




   70 
   69 

   70 







   68 

   68 

   69 
   68 


   68 



   69 
   69 

   69 







   57 


   57 
   56 





   57 
   57 

   57 









   33 


   10 
















    8 



   26 



    8 



    1 








   61 








   57 



    1 

    5 



    4 

    1 









   57 

   41 


    3 


   42 
   42 



   26 

    8 





   57 



















   94 


















   94 
   94 





   10 

   84 















  117 





   47 

   97 


























    6 


   94 








   94 







    3 




    3 








   92 



    3 










































    4 
    2 



    2 


    3 











    7 







   90 






   91 









   91 



   88 






    5 




   91 


   91 
   10 



   61 
    7 



   68 




   24 



    2 








    8 








    8 


    8 















   32 








    5 
   11 

    5 



   27 





   27 









   27 







   19 




    1 



   18 





   18 



   18 


   18 
    6 
   12 





   15 


   15 


   15 










    6 











    6 



    5 











    3 





1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
// SPDX-License-Identifier: GPL-2.0
/*
 *        linux/mm/mlock.c
 *
 *  (C) Copyright 1995 Linus Torvalds
 *  (C) Copyright 2002 Christoph Hellwig
 */

#include <linux/capability.h>
#include <linux/mman.h>
#include <linux/mm.h>
#include <linux/sched/user.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
#include <linux/pagewalk.h>
#include <linux/mempolicy.h>
#include <linux/syscalls.h>
#include <linux/sched.h>
#include <linux/export.h>
#include <linux/rmap.h>
#include <linux/mmzone.h>
#include <linux/hugetlb.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/secretmem.h>

#include "internal.h"

struct mlock_fbatch {
        local_lock_t lock;
        struct folio_batch fbatch;
};

static DEFINE_PER_CPU(struct mlock_fbatch, mlock_fbatch) = {
        .lock = INIT_LOCAL_LOCK(lock),
};

bool can_do_mlock(void)
{
        if (rlimit(RLIMIT_MEMLOCK) != 0)
                return true;
        if (capable(CAP_IPC_LOCK))
                return true;
        return false;
}
EXPORT_SYMBOL(can_do_mlock);

/*
 * Mlocked folios are marked with the PG_mlocked flag for efficient testing
 * in vmscan and, possibly, the fault path; and to support semi-accurate
 * statistics.
 *
 * An mlocked folio [folio_test_mlocked(folio)] is unevictable.  As such, it
 * will be ostensibly placed on the LRU "unevictable" list (actually no such
 * list exists), rather than the [in]active lists. PG_unevictable is set to
 * indicate the unevictable state.
 */

static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec)
{
        /* There is nothing more we can do while it's off LRU */
        if (!folio_test_clear_lru(folio))
                return lruvec;

        lruvec = folio_lruvec_relock_irq(folio, lruvec);

        if (unlikely(folio_evictable(folio))) {
                /*
                 * This is a little surprising, but quite possible: PG_mlocked
                 * must have got cleared already by another CPU.  Could this
                 * folio be unevictable?  I'm not sure, but move it now if so.
                 */
                if (folio_test_unevictable(folio)) {
                        lruvec_del_folio(lruvec, folio);
                        folio_clear_unevictable(folio);
                        lruvec_add_folio(lruvec, folio);

                        __count_vm_events(UNEVICTABLE_PGRESCUED,
                                          folio_nr_pages(folio));
                }
                goto out;
        }

        if (folio_test_unevictable(folio)) {
                if (folio_test_mlocked(folio))
                        folio->mlock_count++;
                goto out;
        }

        lruvec_del_folio(lruvec, folio);
        folio_clear_active(folio);
        folio_set_unevictable(folio);
        folio->mlock_count = !!folio_test_mlocked(folio);
        lruvec_add_folio(lruvec, folio);
        __count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
out:
        folio_set_lru(folio);
        return lruvec;
}

static struct lruvec *__mlock_new_folio(struct folio *folio, struct lruvec *lruvec)
{
        VM_BUG_ON_FOLIO(folio_test_lru(folio), folio);

        lruvec = folio_lruvec_relock_irq(folio, lruvec);

        /* As above, this is a little surprising, but possible */
        if (unlikely(folio_evictable(folio)))
                goto out;

        folio_set_unevictable(folio);
        folio->mlock_count = !!folio_test_mlocked(folio);
        __count_vm_events(UNEVICTABLE_PGCULLED, folio_nr_pages(folio));
out:
        lruvec_add_folio(lruvec, folio);
        folio_set_lru(folio);
        return lruvec;
}

static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec)
{
        int nr_pages = folio_nr_pages(folio);
        bool isolated = false;

        if (!folio_test_clear_lru(folio))
                goto munlock;

        isolated = true;
        lruvec = folio_lruvec_relock_irq(folio, lruvec);

        if (folio_test_unevictable(folio)) {
                /* Then mlock_count is maintained, but might undercount */
                if (folio->mlock_count)
                        folio->mlock_count--;
                if (folio->mlock_count)
                        goto out;
        }
        /* else assume that was the last mlock: reclaim will fix it if not */

munlock:
        if (folio_test_clear_mlocked(folio)) {
                __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages);
                if (isolated || !folio_test_unevictable(folio))
                        __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages);
                else
                        __count_vm_events(UNEVICTABLE_PGSTRANDED, nr_pages);
        }

        /* folio_evictable() has to be checked *after* clearing Mlocked */
        if (isolated && folio_test_unevictable(folio) && folio_evictable(folio)) {
                lruvec_del_folio(lruvec, folio);
                folio_clear_unevictable(folio);
                lruvec_add_folio(lruvec, folio);
                __count_vm_events(UNEVICTABLE_PGRESCUED, nr_pages);
        }
out:
        if (isolated)
                folio_set_lru(folio);
        return lruvec;
}

/*
 * Flags held in the low bits of a struct folio pointer on the mlock_fbatch.
 */
#define LRU_FOLIO 0x1
#define NEW_FOLIO 0x2
static inline struct folio *mlock_lru(struct folio *folio)
{
        return (struct folio *)((unsigned long)folio + LRU_FOLIO);
}

static inline struct folio *mlock_new(struct folio *folio)
{
        return (struct folio *)((unsigned long)folio + NEW_FOLIO);
}

/*
 * mlock_folio_batch() is derived from folio_batch_move_lru(): perhaps that can
 * make use of such folio pointer flags in future, but for now just keep it for
 * mlock.  We could use three separate folio batches instead, but one feels
 * better (munlocking a full folio batch does not need to drain mlocking folio
 * batches first).
 */
static void mlock_folio_batch(struct folio_batch *fbatch)
{
        struct lruvec *lruvec = NULL;
        unsigned long mlock;
        struct folio *folio;
        int i;

        for (i = 0; i < folio_batch_count(fbatch); i++) {
                folio = fbatch->folios[i];
                mlock = (unsigned long)folio & (LRU_FOLIO | NEW_FOLIO);
                folio = (struct folio *)((unsigned long)folio - mlock);
                fbatch->folios[i] = folio;

                if (mlock & LRU_FOLIO)
                        lruvec = __mlock_folio(folio, lruvec);
                else if (mlock & NEW_FOLIO)
                        lruvec = __mlock_new_folio(folio, lruvec);
                else
                        lruvec = __munlock_folio(folio, lruvec);
        }

        if (lruvec)
                unlock_page_lruvec_irq(lruvec);
        folios_put(fbatch);
}

void mlock_drain_local(void)
{
        struct folio_batch *fbatch;

        local_lock(&mlock_fbatch.lock);
        fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
        if (folio_batch_count(fbatch))
                mlock_folio_batch(fbatch);
        local_unlock(&mlock_fbatch.lock);
}

void mlock_drain_remote(int cpu)
{
        struct folio_batch *fbatch;

        WARN_ON_ONCE(cpu_online(cpu));
        fbatch = &per_cpu(mlock_fbatch.fbatch, cpu);
        if (folio_batch_count(fbatch))
                mlock_folio_batch(fbatch);
}

bool need_mlock_drain(int cpu)
{
        return folio_batch_count(&per_cpu(mlock_fbatch.fbatch, cpu));
}

/**
 * mlock_folio - mlock a folio already on (or temporarily off) LRU
 * @folio: folio to be mlocked.
 */
void mlock_folio(struct folio *folio)
{
        struct folio_batch *fbatch;

        local_lock(&mlock_fbatch.lock);
        fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);

        if (!folio_test_set_mlocked(folio)) {
                int nr_pages = folio_nr_pages(folio);

                zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
                __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);
        }

        folio_get(folio);
        if (!folio_batch_add(fbatch, mlock_lru(folio)) ||
            folio_test_large(folio) || lru_cache_disabled())
                mlock_folio_batch(fbatch);
        local_unlock(&mlock_fbatch.lock);
}

/**
 * mlock_new_folio - mlock a newly allocated folio not yet on LRU
 * @folio: folio to be mlocked, either normal or a THP head.
 */
void mlock_new_folio(struct folio *folio)
{
        struct folio_batch *fbatch;
        int nr_pages = folio_nr_pages(folio);

        local_lock(&mlock_fbatch.lock);
        fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
        folio_set_mlocked(folio);

        zone_stat_mod_folio(folio, NR_MLOCK, nr_pages);
        __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages);

        folio_get(folio);
        if (!folio_batch_add(fbatch, mlock_new(folio)) ||
            folio_test_large(folio) || lru_cache_disabled())
                mlock_folio_batch(fbatch);
        local_unlock(&mlock_fbatch.lock);
}

/**
 * munlock_folio - munlock a folio
 * @folio: folio to be munlocked, either normal or a THP head.
 */
void munlock_folio(struct folio *folio)
{
        struct folio_batch *fbatch;

        local_lock(&mlock_fbatch.lock);
        fbatch = this_cpu_ptr(&mlock_fbatch.fbatch);
        /*
         * folio_test_clear_mlocked(folio) must be left to __munlock_folio(),
         * which will check whether the folio is multiply mlocked.
         */
        folio_get(folio);
        if (!folio_batch_add(fbatch, folio) ||
            folio_test_large(folio) || lru_cache_disabled())
                mlock_folio_batch(fbatch);
        local_unlock(&mlock_fbatch.lock);
}

static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
{
        const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
        unsigned int count = (end - addr) >> PAGE_SHIFT;
        pte_t ptent = ptep_get(pte);

        if (!folio_test_large(folio))
                return 1;

        return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
                               NULL, NULL);
}

static inline bool allow_mlock_munlock(struct folio *folio,
                struct vm_area_struct *vma, unsigned long start,
                unsigned long end, unsigned int step)
{
        /*
         * For unlock, allow munlock large folio which is partially
         * mapped to VMA. As it's possible that large folio is
         * mlocked and VMA is split later.
         *
         * During memory pressure, such kind of large folio can
         * be split. And the pages are not in VM_LOCKed VMA
         * can be reclaimed.
         */
        if (!(vma->vm_flags & VM_LOCKED))
                return true;

        /* folio_within_range() cannot take KSM, but any small folio is OK */
        if (!folio_test_large(folio))
                return true;

        /* folio not in range [start, end), skip mlock */
        if (!folio_within_range(folio, vma, start, end))
                return false;

        /* folio is not fully mapped, skip mlock */
        if (step != folio_nr_pages(folio))
                return false;

        return true;
}

static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
                           unsigned long end, struct mm_walk *walk)

{
        struct vm_area_struct *vma = walk->vma;
        spinlock_t *ptl;
        pte_t *start_pte, *pte;
        pte_t ptent;
        struct folio *folio;
        unsigned int step = 1;
        unsigned long start = addr;

        ptl = pmd_trans_huge_lock(pmd, vma);
        if (ptl) {
                if (!pmd_present(*pmd))
                        goto out;
                if (is_huge_zero_pmd(*pmd))
                        goto out;
                folio = pmd_folio(*pmd);
                if (folio_is_zone_device(folio))
                        goto out;
                if (vma->vm_flags & VM_LOCKED)
                        mlock_folio(folio);
                else
                        munlock_folio(folio);
                goto out;
        }

        start_pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
        if (!start_pte) {
                walk->action = ACTION_AGAIN;
                return 0;
        }

        for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) {
                ptent = ptep_get(pte);
                if (!pte_present(ptent))
                        continue;
                folio = vm_normal_folio(vma, addr, ptent);
                if (!folio || folio_is_zone_device(folio))
                        continue;

                step = folio_mlock_step(folio, pte, addr, end);
                if (!allow_mlock_munlock(folio, vma, start, end, step))
                        goto next_entry;

                if (vma->vm_flags & VM_LOCKED)
                        mlock_folio(folio);
                else
                        munlock_folio(folio);

next_entry:
                pte += step - 1;
                addr += (step - 1) << PAGE_SHIFT;
        }
        pte_unmap(start_pte);
out:
        spin_unlock(ptl);
        cond_resched();
        return 0;
}

/*
 * mlock_vma_pages_range() - mlock any pages already in the range,
 *                           or munlock all pages in the range.
 * @vma - vma containing range to be mlock()ed or munlock()ed
 * @start - start address in @vma of the range
 * @end - end of range in @vma
 * @newflags - the new set of flags for @vma.
 *
 * Called for mlock(), mlock2() and mlockall(), to set @vma VM_LOCKED;
 * called for munlock() and munlockall(), to clear VM_LOCKED from @vma.
 */
static void mlock_vma_pages_range(struct vm_area_struct *vma,
        unsigned long start, unsigned long end, vm_flags_t newflags)
{
        static const struct mm_walk_ops mlock_walk_ops = {
                .pmd_entry = mlock_pte_range,
                .walk_lock = PGWALK_WRLOCK_VERIFY,
        };

        /*
         * There is a slight chance that concurrent page migration,
         * or page reclaim finding a page of this now-VM_LOCKED vma,
         * will call mlock_vma_folio() and raise page's mlock_count:
         * double counting, leaving the page unevictable indefinitely.
         * Communicate this danger to mlock_vma_folio() with VM_IO,
         * which is a VM_SPECIAL flag not allowed on VM_LOCKED vmas.
         * mmap_lock is held in write mode here, so this weird
         * combination should not be visible to other mmap_lock users;
         * but WRITE_ONCE so rmap walkers must see VM_IO if VM_LOCKED.
         */
        if (newflags & VM_LOCKED)
                newflags |= VM_IO;
        vma_start_write(vma);
        vm_flags_reset_once(vma, newflags);

        lru_add_drain();
        walk_page_range(vma->vm_mm, start, end, &mlock_walk_ops, NULL);
        lru_add_drain();

        if (newflags & VM_IO) {
                newflags &= ~VM_IO;
                vm_flags_reset_once(vma, newflags);
        }
}

/*
 * mlock_fixup  - handle mlock[all]/munlock[all] requests.
 *
 * Filters out "special" vmas -- VM_LOCKED never gets set for these, and
 * munlock is a no-op.  However, for some special vmas, we go ahead and
 * populate the ptes.
 *
 * For vmas that pass the filters, merge/split as appropriate.
 */
static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma,
               struct vm_area_struct **prev, unsigned long start,
               unsigned long end, vm_flags_t newflags)
{
        struct mm_struct *mm = vma->vm_mm;
        int nr_pages;
        int ret = 0;
        vm_flags_t oldflags = vma->vm_flags;

        if (newflags == oldflags || (oldflags & VM_SPECIAL) ||
            is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm) ||
            vma_is_dax(vma) || vma_is_secretmem(vma) || (oldflags & VM_DROPPABLE))
                /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */
                goto out;

        vma = vma_modify_flags(vmi, *prev, vma, start, end, newflags);
        if (IS_ERR(vma)) {
                ret = PTR_ERR(vma);
                goto out;
        }

        /*
         * Keep track of amount of locked VM.
         */
        nr_pages = (end - start) >> PAGE_SHIFT;
        if (!(newflags & VM_LOCKED))
                nr_pages = -nr_pages;
        else if (oldflags & VM_LOCKED)
                nr_pages = 0;
        mm->locked_vm += nr_pages;

        /*
         * vm_flags is protected by the mmap_lock held in write mode.
         * It's okay if try_to_unmap_one unmaps a page just after we
         * set VM_LOCKED, populate_vma_page_range will bring it back.
         */
        if ((newflags & VM_LOCKED) && (oldflags & VM_LOCKED)) {
                /* No work to do, and mlocking twice would be wrong */
                vma_start_write(vma);
                vm_flags_reset(vma, newflags);
        } else {
                mlock_vma_pages_range(vma, start, end, newflags);
        }
out:
        *prev = vma;
        return ret;
}

static int apply_vma_lock_flags(unsigned long start, size_t len,
                                vm_flags_t flags)
{
        unsigned long nstart, end, tmp;
        struct vm_area_struct *vma, *prev;
        VMA_ITERATOR(vmi, current->mm, start);

        VM_BUG_ON(offset_in_page(start));
        VM_BUG_ON(len != PAGE_ALIGN(len));
        end = start + len;
        if (end < start)
                return -EINVAL;
        if (end == start)
                return 0;
        vma = vma_iter_load(&vmi);
        if (!vma)
                return -ENOMEM;

        prev = vma_prev(&vmi);
        if (start > vma->vm_start)
                prev = vma;

        nstart = start;
        tmp = vma->vm_start;
        for_each_vma_range(vmi, vma, end) {
                int error;
                vm_flags_t newflags;

                if (vma->vm_start != tmp)
                        return -ENOMEM;

                newflags = vma->vm_flags & ~VM_LOCKED_MASK;
                newflags |= flags;
                /* Here we know that  vma->vm_start <= nstart < vma->vm_end. */
                tmp = vma->vm_end;
                if (tmp > end)
                        tmp = end;
                error = mlock_fixup(&vmi, vma, &prev, nstart, tmp, newflags);
                if (error)
                        return error;
                tmp = vma_iter_end(&vmi);
                nstart = tmp;
        }

        if (tmp < end)
                return -ENOMEM;

        return 0;
}

/*
 * Go through vma areas and sum size of mlocked
 * vma pages, as return value.
 * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT)
 * is also counted.
 * Return value: previously mlocked page counts
 */
static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm,
                unsigned long start, size_t len)
{
        struct vm_area_struct *vma;
        unsigned long count = 0;
        unsigned long end;
        VMA_ITERATOR(vmi, mm, start);

        /* Don't overflow past ULONG_MAX */
        if (unlikely(ULONG_MAX - len < start))
                end = ULONG_MAX;
        else
                end = start + len;

        for_each_vma_range(vmi, vma, end) {
                if (vma->vm_flags & VM_LOCKED) {
                        if (start > vma->vm_start)
                                count -= (start - vma->vm_start);
                        if (end < vma->vm_end) {
                                count += end - vma->vm_start;
                                break;
                        }
                        count += vma->vm_end - vma->vm_start;
                }
        }

        return count >> PAGE_SHIFT;
}

/*
 * convert get_user_pages() return value to posix mlock() error
 */
static int __mlock_posix_error_return(long retval)
{
        if (retval == -EFAULT)
                retval = -ENOMEM;
        else if (retval == -ENOMEM)
                retval = -EAGAIN;
        return retval;
}

static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
{
        unsigned long locked;
        unsigned long lock_limit;
        int error = -ENOMEM;

        start = untagged_addr(start);

        if (!can_do_mlock())
                return -EPERM;

        len = PAGE_ALIGN(len + (offset_in_page(start)));
        start &= PAGE_MASK;

        lock_limit = rlimit(RLIMIT_MEMLOCK);
        lock_limit >>= PAGE_SHIFT;
        locked = len >> PAGE_SHIFT;

        if (mmap_write_lock_killable(current->mm))
                return -EINTR;

        locked += current->mm->locked_vm;
        if ((locked > lock_limit) && (!capable(CAP_IPC_LOCK))) {
                /*
                 * It is possible that the regions requested intersect with
                 * previously mlocked areas, that part area in "mm->locked_vm"
                 * should not be counted to new mlock increment count. So check
                 * and adjust locked count if necessary.
                 */
                locked -= count_mm_mlocked_page_nr(current->mm,
                                start, len);
        }

        /* check against resource limits */
        if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
                error = apply_vma_lock_flags(start, len, flags);

        mmap_write_unlock(current->mm);
        if (error)
                return error;

        error = __mm_populate(start, len, 0);
        if (error)
                return __mlock_posix_error_return(error);
        return 0;
}

SYSCALL_DEFINE2(mlock, unsigned long, start, size_t, len)
{
        return do_mlock(start, len, VM_LOCKED);
}

SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
{
        vm_flags_t vm_flags = VM_LOCKED;

        if (flags & ~MLOCK_ONFAULT)
                return -EINVAL;

        if (flags & MLOCK_ONFAULT)
                vm_flags |= VM_LOCKONFAULT;

        return do_mlock(start, len, vm_flags);
}

SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
{
        int ret;

        start = untagged_addr(start);

        len = PAGE_ALIGN(len + (offset_in_page(start)));
        start &= PAGE_MASK;

        if (mmap_write_lock_killable(current->mm))
                return -EINTR;
        ret = apply_vma_lock_flags(start, len, 0);
        mmap_write_unlock(current->mm);

        return ret;
}

/*
 * Take the MCL_* flags passed into mlockall (or 0 if called from munlockall)
 * and translate into the appropriate modifications to mm->def_flags and/or the
 * flags for all current VMAs.
 *
 * There are a couple of subtleties with this.  If mlockall() is called multiple
 * times with different flags, the values do not necessarily stack.  If mlockall
 * is called once including the MCL_FUTURE flag and then a second time without
 * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
 */
static int apply_mlockall_flags(int flags)
{
        VMA_ITERATOR(vmi, current->mm, 0);
        struct vm_area_struct *vma, *prev = NULL;
        vm_flags_t to_add = 0;

        current->mm->def_flags &= ~VM_LOCKED_MASK;
        if (flags & MCL_FUTURE) {
                current->mm->def_flags |= VM_LOCKED;

                if (flags & MCL_ONFAULT)
                        current->mm->def_flags |= VM_LOCKONFAULT;

                if (!(flags & MCL_CURRENT))
                        goto out;
        }

        if (flags & MCL_CURRENT) {
                to_add |= VM_LOCKED;
                if (flags & MCL_ONFAULT)
                        to_add |= VM_LOCKONFAULT;
        }

        for_each_vma(vmi, vma) {
                int error;
                vm_flags_t newflags;

                newflags = vma->vm_flags & ~VM_LOCKED_MASK;
                newflags |= to_add;

                error = mlock_fixup(&vmi, vma, &prev, vma->vm_start, vma->vm_end,
                                    newflags);
                /* Ignore errors, but prev needs fixing up. */
                if (error)
                        prev = vma;
                cond_resched();
        }
out:
        return 0;
}

SYSCALL_DEFINE1(mlockall, int, flags)
{
        unsigned long lock_limit;
        int ret;

        if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)) ||
            flags == MCL_ONFAULT)
                return -EINVAL;

        if (!can_do_mlock())
                return -EPERM;

        lock_limit = rlimit(RLIMIT_MEMLOCK);
        lock_limit >>= PAGE_SHIFT;

        if (mmap_write_lock_killable(current->mm))
                return -EINTR;

        ret = -ENOMEM;
        if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
            capable(CAP_IPC_LOCK))
                ret = apply_mlockall_flags(flags);
        mmap_write_unlock(current->mm);
        if (!ret && (flags & MCL_CURRENT))
                mm_populate(0, TASK_SIZE);

        return ret;
}

SYSCALL_DEFINE0(munlockall)
{
        int ret;

        if (mmap_write_lock_killable(current->mm))
                return -EINTR;
        ret = apply_mlockall_flags(0);
        mmap_write_unlock(current->mm);
        return ret;
}

/*
 * Objects with different lifetime than processes (SHM_LOCK and SHM_HUGETLB
 * shm segments) get accounted against the user_struct instead.
 */
static DEFINE_SPINLOCK(shmlock_user_lock);

int user_shm_lock(size_t size, struct ucounts *ucounts)
{
        unsigned long lock_limit, locked;
        long memlock;
        int allowed = 0;

        locked = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
        lock_limit = rlimit(RLIMIT_MEMLOCK);
        if (lock_limit != RLIM_INFINITY)
                lock_limit >>= PAGE_SHIFT;
        spin_lock(&shmlock_user_lock);
        memlock = inc_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);

        if ((memlock == LONG_MAX || memlock > lock_limit) && !capable(CAP_IPC_LOCK)) {
                dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
                goto out;
        }
        if (!get_ucounts(ucounts)) {
                dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, locked);
                allowed = 0;
                goto out;
        }
        allowed = 1;
out:
        spin_unlock(&shmlock_user_lock);
        return allowed;
}

void user_shm_unlock(size_t size, struct ucounts *ucounts)
{
        spin_lock(&shmlock_user_lock);
        dec_rlimit_ucounts(ucounts, UCOUNT_RLIMIT_MEMLOCK, (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
        spin_unlock(&shmlock_user_lock);
        put_ucounts(ucounts);
}




















































































































































































































































  241 
  240 













  237 






  238 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (c) 2018-2024 Oracle.  All Rights Reserved.
 * Author: Darrick J. Wong <djwong@kernel.org>
 */
#include "xfs.h"
#include "xfs_fs.h"
#include "xfs_shared.h"
#include "xfs_format.h"
#include "xfs_log_format.h"
#include "xfs_trans_resv.h"
#include "xfs_bit.h"
#include "xfs_sb.h"
#include "xfs_mount.h"
#include "xfs_defer.h"
#include "xfs_trans.h"
#include "xfs_metafile.h"
#include "xfs_trace.h"
#include "xfs_inode.h"
#include "xfs_quota.h"
#include "xfs_errortag.h"
#include "xfs_error.h"
#include "xfs_alloc.h"
#include "xfs_rtgroup.h"
#include "xfs_rtrmap_btree.h"
#include "xfs_rtrefcount_btree.h"

static const struct {
        enum xfs_metafile_type        mtype;
        const char                *name;
} xfs_metafile_type_strs[] = { XFS_METAFILE_TYPE_STR };

const char *
xfs_metafile_type_str(enum xfs_metafile_type metatype)
{
        unsigned int        i;

        for (i = 0; i < ARRAY_SIZE(xfs_metafile_type_strs); i++) {
                if (xfs_metafile_type_strs[i].mtype == metatype)
                        return xfs_metafile_type_strs[i].name;
        }

        return NULL;
}

/* Set up an inode to be recognized as a metadata directory inode. */
void
xfs_metafile_set_iflag(
        struct xfs_trans        *tp,
        struct xfs_inode        *ip,
        enum xfs_metafile_type        metafile_type)
{
        VFS_I(ip)->i_mode &= ~0777;
        VFS_I(ip)->i_uid = GLOBAL_ROOT_UID;
        VFS_I(ip)->i_gid = GLOBAL_ROOT_GID;
        if (S_ISDIR(VFS_I(ip)->i_mode))
                ip->i_diflags |= XFS_METADIR_DIFLAGS;
        else
                ip->i_diflags |= XFS_METAFILE_DIFLAGS;
        ip->i_diflags2 &= ~XFS_DIFLAG2_DAX;
        ip->i_diflags2 |= XFS_DIFLAG2_METADATA;
        ip->i_metatype = metafile_type;
        xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
}

/* Clear the metadata directory inode flag. */
void
xfs_metafile_clear_iflag(
        struct xfs_trans        *tp,
        struct xfs_inode        *ip)
{
        ASSERT(xfs_is_metadir_inode(ip));
        ASSERT(VFS_I(ip)->i_nlink == 0);

        ip->i_diflags2 &= ~XFS_DIFLAG2_METADATA;
        xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
}

/*
 * Is the metafile reservations at or beneath a certain threshold?
 */
static inline bool
xfs_metafile_resv_can_cover(
        struct xfs_mount        *mp,
        int64_t                        rhs)
{
        /*
         * The amount of space that can be allocated to this metadata file is
         * the remaining reservation for the particular metadata file + the
         * global free block count.  Take care of the first case to avoid
         * touching the per-cpu counter.
         */
        if (mp->m_metafile_resv_avail >= rhs)
                return true;

        /*
         * There aren't enough blocks left in the inode's reservation, but it
         * isn't critical unless there also isn't enough free space.
         */
        return xfs_compare_freecounter(mp, XC_FREE_BLOCKS,
                        rhs - mp->m_metafile_resv_avail, 2048) >= 0;
}

/*
 * Is the metafile reservation critically low on blocks?  For now we'll define
 * that as the number of blocks we can get our hands on being less than 10% of
 * what we reserved or less than some arbitrary number (maximum btree height).
 */
bool
xfs_metafile_resv_critical(
        struct xfs_mount        *mp)
{
        ASSERT(xfs_has_metadir(mp));

        trace_xfs_metafile_resv_critical(mp, 0);

        if (!xfs_metafile_resv_can_cover(mp, mp->m_rtbtree_maxlevels))
                return true;

        if (!xfs_metafile_resv_can_cover(mp,
                        div_u64(mp->m_metafile_resv_target, 10)))
                return true;

        return XFS_TEST_ERROR(false, mp, XFS_ERRTAG_METAFILE_RESV_CRITICAL);
}

/* Allocate a block from the metadata file's reservation. */
void
xfs_metafile_resv_alloc_space(
        struct xfs_inode        *ip,
        struct xfs_alloc_arg        *args)
{
        struct xfs_mount        *mp = ip->i_mount;
        int64_t                        len = args->len;

        ASSERT(xfs_is_metadir_inode(ip));
        ASSERT(args->resv == XFS_AG_RESV_METAFILE);

        trace_xfs_metafile_resv_alloc_space(mp, args->len);

        /*
         * Allocate the blocks from the metadata inode's block reservation
         * and update the ondisk sb counter.
         */
        mutex_lock(&mp->m_metafile_resv_lock);
        if (mp->m_metafile_resv_avail > 0) {
                int64_t                from_resv;

                from_resv = min_t(int64_t, len, mp->m_metafile_resv_avail);
                mp->m_metafile_resv_avail -= from_resv;
                xfs_mod_delalloc(ip, 0, -from_resv);
                xfs_trans_mod_sb(args->tp, XFS_TRANS_SB_RES_FDBLOCKS,
                                -from_resv);
                len -= from_resv;
        }

        /*
         * Any allocation in excess of the reservation requires in-core and
         * on-disk fdblocks updates.  If we can grab @len blocks from the
         * in-core fdblocks then all we need to do is update the on-disk
         * superblock; if not, then try to steal some from the transaction's
         * block reservation.  Overruns are only expected for rmap btrees.
         */
        if (len) {
                unsigned int        field;
                int                error;

                error = xfs_dec_fdblocks(ip->i_mount, len, true);
                if (error)
                        field = XFS_TRANS_SB_FDBLOCKS;
                else
                        field = XFS_TRANS_SB_RES_FDBLOCKS;

                xfs_trans_mod_sb(args->tp, field, -len);
        }

        mp->m_metafile_resv_used += args->len;
        mutex_unlock(&mp->m_metafile_resv_lock);

        ip->i_nblocks += args->len;
        xfs_trans_log_inode(args->tp, ip, XFS_ILOG_CORE);
}

/* Free a block to the metadata file's reservation. */
void
xfs_metafile_resv_free_space(
        struct xfs_inode        *ip,
        struct xfs_trans        *tp,
        xfs_filblks_t                len)
{
        struct xfs_mount        *mp = ip->i_mount;
        int64_t                        to_resv;

        ASSERT(xfs_is_metadir_inode(ip));

        trace_xfs_metafile_resv_free_space(mp, len);

        ip->i_nblocks -= len;
        xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);

        mutex_lock(&mp->m_metafile_resv_lock);
        mp->m_metafile_resv_used -= len;

        /*
         * Add the freed blocks back into the inode's delalloc reservation
         * until it reaches the maximum size.  Update the ondisk fdblocks only.
         */
        to_resv = mp->m_metafile_resv_target -
                (mp->m_metafile_resv_used + mp->m_metafile_resv_avail);
        if (to_resv > 0) {
                to_resv = min_t(int64_t, to_resv, len);
                mp->m_metafile_resv_avail += to_resv;
                xfs_mod_delalloc(ip, 0, to_resv);
                xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FDBLOCKS, to_resv);
                len -= to_resv;
        }
        mutex_unlock(&mp->m_metafile_resv_lock);

        /*
         * Everything else goes back to the filesystem, so update the in-core
         * and on-disk counters.
         */
        if (len)
                xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, len);
}

static void
__xfs_metafile_resv_free(
        struct xfs_mount        *mp)
{
        if (mp->m_metafile_resv_avail) {
                xfs_mod_sb_delalloc(mp, -(int64_t)mp->m_metafile_resv_avail);
                xfs_add_fdblocks(mp, mp->m_metafile_resv_avail);
        }
        mp->m_metafile_resv_avail = 0;
        mp->m_metafile_resv_used = 0;
        mp->m_metafile_resv_target = 0;
}

/* Release unused metafile space reservation. */
void
xfs_metafile_resv_free(
        struct xfs_mount        *mp)
{
        if (!xfs_has_metadir(mp))
                return;

        trace_xfs_metafile_resv_free(mp, 0);

        mutex_lock(&mp->m_metafile_resv_lock);
        __xfs_metafile_resv_free(mp);
        mutex_unlock(&mp->m_metafile_resv_lock);
}

/* Set up a metafile space reservation. */
int
xfs_metafile_resv_init(
        struct xfs_mount        *mp)
{
        struct xfs_rtgroup        *rtg = NULL;
        xfs_filblks_t                used = 0, target = 0;
        xfs_filblks_t                hidden_space;
        xfs_rfsblock_t                dblocks_avail = mp->m_sb.sb_dblocks / 4;
        int                        error = 0;

        if (!xfs_has_metadir(mp))
                return 0;

        /*
         * Free any previous reservation to have a clean slate.
         */
        mutex_lock(&mp->m_metafile_resv_lock);
        __xfs_metafile_resv_free(mp);

        /*
         * Currently the only btree metafiles that require reservations are the
         * rtrmap and the rtrefcount.  Anything new will have to be added here
         * as well.
         */
        while ((rtg = xfs_rtgroup_next(mp, rtg))) {
                if (xfs_has_rtrmapbt(mp)) {
                        used += rtg_rmap(rtg)->i_nblocks;
                        target += xfs_rtrmapbt_calc_reserves(mp);
                }
                if (xfs_has_rtreflink(mp)) {
                        used += rtg_refcount(rtg)->i_nblocks;
                        target += xfs_rtrefcountbt_calc_reserves(mp);
                }
        }

        if (!target)
                goto out_unlock;

        /*
         * Space taken by the per-AG metadata btrees are accounted on-disk as
         * used space.  We therefore only hide the space that is reserved but
         * not used by the trees.
         */
        if (used > target)
                target = used;
        else if (target > dblocks_avail)
                target = dblocks_avail;
        hidden_space = target - used;

        error = xfs_dec_fdblocks(mp, hidden_space, true);
        if (error) {
                trace_xfs_metafile_resv_init_error(mp, 0);
                goto out_unlock;
        }

        xfs_mod_sb_delalloc(mp, hidden_space);

        mp->m_metafile_resv_target = target;
        mp->m_metafile_resv_used = used;
        mp->m_metafile_resv_avail = hidden_space;

        trace_xfs_metafile_resv_init(mp, target);

out_unlock:
        mutex_unlock(&mp->m_metafile_resv_lock);
        return error;
}


















































































































































































































































































































































































































































































































































    1 

































































































































































































































































































































































































































































































































































































   40 















































   40 




   40 











   40 



















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
// SPDX-License-Identifier: GPL-2.0
#include <linux/types.h>
#include <linux/netfilter.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/skbuff.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/percpu.h>
#include <linux/netdevice.h>
#include <linux/security.h>
#include <net/net_namespace.h>
#ifdef CONFIG_SYSCTL
#include <linux/sysctl.h>
#endif

#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_core.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_expect.h>
#include <net/netfilter/nf_conntrack_helper.h>
#include <net/netfilter/nf_conntrack_acct.h>
#include <net/netfilter/nf_conntrack_zones.h>
#include <net/netfilter/nf_conntrack_timestamp.h>
#include <linux/rculist_nulls.h>

static bool enable_hooks __read_mostly;
MODULE_PARM_DESC(enable_hooks, "Always enable conntrack hooks");
module_param(enable_hooks, bool, 0000);

unsigned int nf_conntrack_net_id __read_mostly;

#ifdef CONFIG_NF_CONNTRACK_PROCFS
void
print_tuple(struct seq_file *s, const struct nf_conntrack_tuple *tuple,
            const struct nf_conntrack_l4proto *l4proto)
{
        switch (tuple->src.l3num) {
        case NFPROTO_IPV4:
                seq_printf(s, "src=%pI4 dst=%pI4 ",
                           &tuple->src.u3.ip, &tuple->dst.u3.ip);
                break;
        case NFPROTO_IPV6:
                seq_printf(s, "src=%pI6 dst=%pI6 ",
                           tuple->src.u3.ip6, tuple->dst.u3.ip6);
                break;
        default:
                break;
        }

        switch (l4proto->l4proto) {
        case IPPROTO_ICMP:
                seq_printf(s, "type=%u code=%u id=%u ",
                           tuple->dst.u.icmp.type,
                           tuple->dst.u.icmp.code,
                           ntohs(tuple->src.u.icmp.id));
                break;
        case IPPROTO_TCP:
                seq_printf(s, "sport=%hu dport=%hu ",
                           ntohs(tuple->src.u.tcp.port),
                           ntohs(tuple->dst.u.tcp.port));
                break;
        case IPPROTO_UDPLITE:
        case IPPROTO_UDP:
                seq_printf(s, "sport=%hu dport=%hu ",
                           ntohs(tuple->src.u.udp.port),
                           ntohs(tuple->dst.u.udp.port));

                break;
        case IPPROTO_DCCP:
                seq_printf(s, "sport=%hu dport=%hu ",
                           ntohs(tuple->src.u.dccp.port),
                           ntohs(tuple->dst.u.dccp.port));
                break;
        case IPPROTO_SCTP:
                seq_printf(s, "sport=%hu dport=%hu ",
                           ntohs(tuple->src.u.sctp.port),
                           ntohs(tuple->dst.u.sctp.port));
                break;
        case IPPROTO_ICMPV6:
                seq_printf(s, "type=%u code=%u id=%u ",
                           tuple->dst.u.icmp.type,
                           tuple->dst.u.icmp.code,
                           ntohs(tuple->src.u.icmp.id));
                break;
        case IPPROTO_GRE:
                seq_printf(s, "srckey=0x%x dstkey=0x%x ",
                           ntohs(tuple->src.u.gre.key),
                           ntohs(tuple->dst.u.gre.key));
                break;
        default:
                break;
        }
}
EXPORT_SYMBOL_GPL(print_tuple);

struct ct_iter_state {
        struct seq_net_private p;
        struct hlist_nulls_head *hash;
        unsigned int htable_size;
        unsigned int skip_elems;
        unsigned int bucket;
        u_int64_t time_now;
};

static struct nf_conntrack_tuple_hash *ct_get_next(const struct net *net,
                                                   struct ct_iter_state *st)
{
        struct nf_conntrack_tuple_hash *h;
        struct hlist_nulls_node *n;
        unsigned int i;

        for (i = st->bucket; i < st->htable_size; i++) {
                unsigned int skip = 0;

restart:
                hlist_nulls_for_each_entry_rcu(h, n, &st->hash[i], hnnode) {
                        struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
                        struct hlist_nulls_node *tmp = n;

                        if (!net_eq(net, nf_ct_net(ct)))
                                continue;

                        if (++skip <= st->skip_elems)
                                continue;

                        /* h should be returned, skip to nulls marker. */
                        while (!is_a_nulls(tmp))
                                tmp = rcu_dereference(hlist_nulls_next_rcu(tmp));

                        /* check if h is still linked to hash[i] */
                        if (get_nulls_value(tmp) != i) {
                                skip = 0;
                                goto restart;
                        }

                        st->skip_elems = skip;
                        st->bucket = i;
                        return h;
                }

                skip = 0;
                if (get_nulls_value(n) != i)
                        goto restart;

                st->skip_elems = 0;
        }

        st->bucket = i;
        return NULL;
}

static void *ct_seq_start(struct seq_file *seq, loff_t *pos)
        __acquires(RCU)
{
        struct ct_iter_state *st = seq->private;
        struct net *net = seq_file_net(seq);

        st->time_now = ktime_get_real_ns();
        rcu_read_lock();

        nf_conntrack_get_ht(&st->hash, &st->htable_size);

        if (*pos == 0) {
                st->skip_elems = 0;
                st->bucket = 0;
        } else if (st->skip_elems) {
                /* resume from last dumped entry */
                st->skip_elems--;
        }

        return ct_get_next(net, st);
}

static void *ct_seq_next(struct seq_file *s, void *v, loff_t *pos)
{
        struct ct_iter_state *st = s->private;
        struct net *net = seq_file_net(s);

        (*pos)++;
        return ct_get_next(net, st);
}

static void ct_seq_stop(struct seq_file *s, void *v)
        __releases(RCU)
{
        rcu_read_unlock();
}

#ifdef CONFIG_NF_CONNTRACK_SECMARK
static void ct_show_secctx(struct seq_file *s, const struct nf_conn *ct)
{
        struct lsm_context ctx;
        int ret;

        ret = security_secid_to_secctx(ct->secmark, &ctx);
        if (ret < 0)
                return;

        seq_printf(s, "secctx=%s ", ctx.context);

        security_release_secctx(&ctx);
}
#else
static inline void ct_show_secctx(struct seq_file *s, const struct nf_conn *ct)
{
}
#endif

#ifdef CONFIG_NF_CONNTRACK_ZONES
static void ct_show_zone(struct seq_file *s, const struct nf_conn *ct,
                         int dir)
{
        const struct nf_conntrack_zone *zone = nf_ct_zone(ct);

        if (zone->dir != dir)
                return;
        switch (zone->dir) {
        case NF_CT_DEFAULT_ZONE_DIR:
                seq_printf(s, "zone=%u ", zone->id);
                break;
        case NF_CT_ZONE_DIR_ORIG:
                seq_printf(s, "zone-orig=%u ", zone->id);
                break;
        case NF_CT_ZONE_DIR_REPL:
                seq_printf(s, "zone-reply=%u ", zone->id);
                break;
        default:
                break;
        }
}
#else
static inline void ct_show_zone(struct seq_file *s, const struct nf_conn *ct,
                                int dir)
{
}
#endif

#ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
static void ct_show_delta_time(struct seq_file *s, const struct nf_conn *ct)
{
        struct ct_iter_state *st = s->private;
        struct nf_conn_tstamp *tstamp;
        s64 delta_time;

        tstamp = nf_conn_tstamp_find(ct);
        if (tstamp) {
                delta_time = st->time_now - tstamp->start;
                if (delta_time > 0)
                        delta_time = div_s64(delta_time, NSEC_PER_SEC);
                else
                        delta_time = 0;

                seq_printf(s, "delta-time=%llu ",
                           (unsigned long long)delta_time);
        }
        return;
}
#else
static inline void
ct_show_delta_time(struct seq_file *s, const struct nf_conn *ct)
{
}
#endif

static const char* l3proto_name(u16 proto)
{
        switch (proto) {
        case AF_INET: return "ipv4";
        case AF_INET6: return "ipv6";
        }

        return "unknown";
}

static const char* l4proto_name(u16 proto)
{
        switch (proto) {
        case IPPROTO_ICMP: return "icmp";
        case IPPROTO_TCP: return "tcp";
        case IPPROTO_UDP: return "udp";
        case IPPROTO_DCCP: return "dccp";
        case IPPROTO_GRE: return "gre";
        case IPPROTO_SCTP: return "sctp";
        case IPPROTO_UDPLITE: return "udplite";
        case IPPROTO_ICMPV6: return "icmpv6";
        }

        return "unknown";
}

static void
seq_print_acct(struct seq_file *s, const struct nf_conn *ct, int dir)
{
        struct nf_conn_acct *acct;
        struct nf_conn_counter *counter;

        acct = nf_conn_acct_find(ct);
        if (!acct)
                return;

        counter = acct->counter;
        seq_printf(s, "packets=%llu bytes=%llu ",
                   (unsigned long long)atomic64_read(&counter[dir].packets),
                   (unsigned long long)atomic64_read(&counter[dir].bytes));
}

/* return 0 on success, 1 in case of error */
static int ct_seq_show(struct seq_file *s, void *v)
{
        struct nf_conntrack_tuple_hash *hash = v;
        struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(hash);
        const struct nf_conntrack_l4proto *l4proto;
        struct net *net = seq_file_net(s);
        int ret = 0;

        WARN_ON(!ct);
        if (unlikely(!refcount_inc_not_zero(&ct->ct_general.use)))
                return 0;

        /* load ->status after refcount increase */
        smp_acquire__after_ctrl_dep();

        if (nf_ct_should_gc(ct)) {
                nf_ct_kill(ct);
                goto release;
        }

        /* we only want to print DIR_ORIGINAL */
        if (NF_CT_DIRECTION(hash))
                goto release;

        if (!net_eq(nf_ct_net(ct), net))
                goto release;

        l4proto = nf_ct_l4proto_find(nf_ct_protonum(ct));

        ret = -ENOSPC;
        seq_printf(s, "%-8s %u %-8s %u ",
                   l3proto_name(nf_ct_l3num(ct)), nf_ct_l3num(ct),
                   l4proto_name(l4proto->l4proto), nf_ct_protonum(ct));

        if (!test_bit(IPS_OFFLOAD_BIT, &ct->status))
                seq_printf(s, "%ld ", nf_ct_expires(ct)  / HZ);

        if (l4proto->print_conntrack)
                l4proto->print_conntrack(s, ct);

        print_tuple(s, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
                    l4proto);

        ct_show_zone(s, ct, NF_CT_ZONE_DIR_ORIG);

        if (seq_has_overflowed(s))
                goto release;

        seq_print_acct(s, ct, IP_CT_DIR_ORIGINAL);

        if (!(test_bit(IPS_SEEN_REPLY_BIT, &ct->status)))
                seq_puts(s, "[UNREPLIED] ");

        print_tuple(s, &ct->tuplehash[IP_CT_DIR_REPLY].tuple, l4proto);

        ct_show_zone(s, ct, NF_CT_ZONE_DIR_REPL);

        seq_print_acct(s, ct, IP_CT_DIR_REPLY);

        if (test_bit(IPS_HW_OFFLOAD_BIT, &ct->status))
                seq_puts(s, "[HW_OFFLOAD] ");
        else if (test_bit(IPS_OFFLOAD_BIT, &ct->status))
                seq_puts(s, "[OFFLOAD] ");
        else if (test_bit(IPS_ASSURED_BIT, &ct->status))
                seq_puts(s, "[ASSURED] ");

        if (seq_has_overflowed(s))
                goto release;

#if defined(CONFIG_NF_CONNTRACK_MARK)
        seq_printf(s, "mark=%u ", READ_ONCE(ct->mark));
#endif

        ct_show_secctx(s, ct);
        ct_show_zone(s, ct, NF_CT_DEFAULT_ZONE_DIR);
        ct_show_delta_time(s, ct);

        seq_printf(s, "use=%u\n", refcount_read(&ct->ct_general.use));

        if (seq_has_overflowed(s))
                goto release;

        ret = 0;
release:
        nf_ct_put(ct);
        return ret;
}

static const struct seq_operations ct_seq_ops = {
        .start = ct_seq_start,
        .next  = ct_seq_next,
        .stop  = ct_seq_stop,
        .show  = ct_seq_show
};

static void *ct_cpu_seq_start(struct seq_file *seq, loff_t *pos)
{
        struct net *net = seq_file_net(seq);
        int cpu;

        if (*pos == 0)
                return SEQ_START_TOKEN;

        for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu + 1;
                return per_cpu_ptr(net->ct.stat, cpu);
        }

        return NULL;
}

static void *ct_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
        struct net *net = seq_file_net(seq);
        int cpu;

        for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
                if (!cpu_possible(cpu))
                        continue;
                *pos = cpu + 1;
                return per_cpu_ptr(net->ct.stat, cpu);
        }
        (*pos)++;
        return NULL;
}

static void ct_cpu_seq_stop(struct seq_file *seq, void *v)
{
}

static int ct_cpu_seq_show(struct seq_file *seq, void *v)
{
        struct net *net = seq_file_net(seq);
        const struct ip_conntrack_stat *st = v;
        unsigned int nr_conntracks;

        if (v == SEQ_START_TOKEN) {
                seq_puts(seq, "entries  clashres found new invalid ignore delete chainlength insert insert_failed drop early_drop icmp_error  expect_new expect_create expect_delete search_restart\n");
                return 0;
        }

        nr_conntracks = nf_conntrack_count(net);

        seq_printf(seq, "%08x  %08x %08x %08x %08x %08x %08x %08x "
                        "%08x %08x %08x %08x %08x  %08x %08x %08x %08x\n",
                   nr_conntracks,
                   st->clash_resolve,
                   st->found,
                   0,
                   st->invalid,
                   0,
                   0,
                   st->chaintoolong,
                   st->insert,
                   st->insert_failed,
                   st->drop,
                   st->early_drop,
                   st->error,

                   st->expect_new,
                   st->expect_create,
                   st->expect_delete,
                   st->search_restart
                );
        return 0;
}

static const struct seq_operations ct_cpu_seq_ops = {
        .start        = ct_cpu_seq_start,
        .next        = ct_cpu_seq_next,
        .stop        = ct_cpu_seq_stop,
        .show        = ct_cpu_seq_show,
};

static int nf_conntrack_standalone_init_proc(struct net *net)
{
        struct proc_dir_entry *pde;
        kuid_t root_uid;
        kgid_t root_gid;

        pde = proc_create_net("nf_conntrack", 0440, net->proc_net, &ct_seq_ops,
                        sizeof(struct ct_iter_state));
        if (!pde)
                goto out_nf_conntrack;

        root_uid = make_kuid(net->user_ns, 0);
        root_gid = make_kgid(net->user_ns, 0);
        if (uid_valid(root_uid) && gid_valid(root_gid))
                proc_set_user(pde, root_uid, root_gid);

        pde = proc_create_net("nf_conntrack", 0444, net->proc_net_stat,
                        &ct_cpu_seq_ops, sizeof(struct seq_net_private));
        if (!pde)
                goto out_stat_nf_conntrack;
        return 0;

out_stat_nf_conntrack:
        remove_proc_entry("nf_conntrack", net->proc_net);
out_nf_conntrack:
        return -ENOMEM;
}

static void nf_conntrack_standalone_fini_proc(struct net *net)
{
        remove_proc_entry("nf_conntrack", net->proc_net_stat);
        remove_proc_entry("nf_conntrack", net->proc_net);
}
#else
static int nf_conntrack_standalone_init_proc(struct net *net)
{
        return 0;
}

static void nf_conntrack_standalone_fini_proc(struct net *net)
{
}
#endif /* CONFIG_NF_CONNTRACK_PROCFS */

u32 nf_conntrack_count(const struct net *net)
{
        const struct nf_conntrack_net *cnet = nf_ct_pernet(net);

        return atomic_read(&cnet->count);
}
EXPORT_SYMBOL_GPL(nf_conntrack_count);

/* Sysctl support */

#ifdef CONFIG_SYSCTL
/* size the user *wants to set */
static unsigned int nf_conntrack_htable_size_user __read_mostly;

static int
nf_conntrack_hash_sysctl(const struct ctl_table *table, int write,
                         void *buffer, size_t *lenp, loff_t *ppos)
{
        int ret;

        /* module_param hashsize could have changed value */
        nf_conntrack_htable_size_user = nf_conntrack_htable_size;

        ret = proc_dointvec(table, write, buffer, lenp, ppos);
        if (ret < 0 || !write)
                return ret;

        /* update ret, we might not be able to satisfy request */
        ret = nf_conntrack_hash_resize(nf_conntrack_htable_size_user);

        /* update it to the actual value used by conntrack */
        nf_conntrack_htable_size_user = nf_conntrack_htable_size;
        return ret;
}

static struct ctl_table_header *nf_ct_netfilter_header;

enum nf_ct_sysctl_index {
        NF_SYSCTL_CT_MAX,
        NF_SYSCTL_CT_COUNT,
        NF_SYSCTL_CT_BUCKETS,
        NF_SYSCTL_CT_CHECKSUM,
        NF_SYSCTL_CT_LOG_INVALID,
        NF_SYSCTL_CT_EXPECT_MAX,
        NF_SYSCTL_CT_ACCT,
#ifdef CONFIG_NF_CONNTRACK_EVENTS
        NF_SYSCTL_CT_EVENTS,
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
        NF_SYSCTL_CT_TIMESTAMP,
#endif
        NF_SYSCTL_CT_PROTO_TIMEOUT_GENERIC,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_SENT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_RECV,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_ESTABLISHED,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_FIN_WAIT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE_WAIT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_LAST_ACK,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_TIME_WAIT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_RETRANS,
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_UNACK,
#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD,
#endif
        NF_SYSCTL_CT_PROTO_TCP_LOOSE,
        NF_SYSCTL_CT_PROTO_TCP_LIBERAL,
        NF_SYSCTL_CT_PROTO_TCP_IGNORE_INVALID_RST,
        NF_SYSCTL_CT_PROTO_TCP_MAX_RETRANS,
        NF_SYSCTL_CT_PROTO_TIMEOUT_UDP,
        NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM,
#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD,
#endif
        NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP,
        NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6,
#ifdef CONFIG_NF_CT_PROTO_SCTP
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_CLOSED,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_WAIT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_ECHOED,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_ESTABLISHED,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_SENT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_RECD,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_ACK_SENT,
        NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_HEARTBEAT_SENT,
#endif
#ifdef CONFIG_NF_CT_PROTO_DCCP
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_REQUEST,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_RESPOND,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_PARTOPEN,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_OPEN,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSEREQ,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSING,
        NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_TIMEWAIT,
        NF_SYSCTL_CT_PROTO_DCCP_LOOSE,
#endif
#ifdef CONFIG_NF_CT_PROTO_GRE
        NF_SYSCTL_CT_PROTO_TIMEOUT_GRE,
        NF_SYSCTL_CT_PROTO_TIMEOUT_GRE_STREAM,
#endif

        NF_SYSCTL_CT_LAST_SYSCTL,
};

static struct ctl_table nf_ct_sysctl_table[] = {
        [NF_SYSCTL_CT_MAX] = {
                .procname        = "nf_conntrack_max",
                .data                = &nf_conntrack_max,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_INT_MAX,
        },
        [NF_SYSCTL_CT_COUNT] = {
                .procname        = "nf_conntrack_count",
                .maxlen                = sizeof(int),
                .mode                = 0444,
                .proc_handler        = proc_dointvec,
        },
        [NF_SYSCTL_CT_BUCKETS] = {
                .procname       = "nf_conntrack_buckets",
                .data           = &nf_conntrack_htable_size_user,
                .maxlen         = sizeof(unsigned int),
                .mode           = 0644,
                .proc_handler   = nf_conntrack_hash_sysctl,
        },
        [NF_SYSCTL_CT_CHECKSUM] = {
                .procname        = "nf_conntrack_checksum",
                .data                = &init_net.ct.sysctl_checksum,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
        [NF_SYSCTL_CT_LOG_INVALID] = {
                .procname        = "nf_conntrack_log_invalid",
                .data                = &init_net.ct.sysctl_log_invalid,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        [NF_SYSCTL_CT_EXPECT_MAX] = {
                .procname        = "nf_conntrack_expect_max",
                .data                = &nf_ct_expect_max,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ONE,
                .extra2                = SYSCTL_INT_MAX,
        },
        [NF_SYSCTL_CT_ACCT] = {
                .procname        = "nf_conntrack_acct",
                .data                = &init_net.ct.sysctl_acct,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
#ifdef CONFIG_NF_CONNTRACK_EVENTS
        [NF_SYSCTL_CT_EVENTS] = {
                .procname        = "nf_conntrack_events",
                .data                = &init_net.ct.sysctl_events,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2                = SYSCTL_TWO,
        },
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
        [NF_SYSCTL_CT_TIMESTAMP] = {
                .procname        = "nf_conntrack_timestamp",
                .data                = &init_net.ct.sysctl_tstamp,
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
#endif
        [NF_SYSCTL_CT_PROTO_TIMEOUT_GENERIC] = {
                .procname        = "nf_conntrack_generic_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_SENT] = {
                .procname        = "nf_conntrack_tcp_timeout_syn_sent",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_SYN_RECV] = {
                .procname        = "nf_conntrack_tcp_timeout_syn_recv",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_ESTABLISHED] = {
                .procname        = "nf_conntrack_tcp_timeout_established",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_FIN_WAIT] = {
                .procname        = "nf_conntrack_tcp_timeout_fin_wait",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE_WAIT] = {
                .procname        = "nf_conntrack_tcp_timeout_close_wait",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_LAST_ACK] = {
                .procname        = "nf_conntrack_tcp_timeout_last_ack",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_TIME_WAIT] = {
                .procname        = "nf_conntrack_tcp_timeout_time_wait",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE] = {
                .procname        = "nf_conntrack_tcp_timeout_close",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_RETRANS] = {
                .procname        = "nf_conntrack_tcp_timeout_max_retrans",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_UNACK] = {
                .procname        = "nf_conntrack_tcp_timeout_unacknowledged",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD] = {
                .procname        = "nf_flowtable_tcp_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#endif
        [NF_SYSCTL_CT_PROTO_TCP_LOOSE] = {
                .procname        = "nf_conntrack_tcp_loose",
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
        [NF_SYSCTL_CT_PROTO_TCP_LIBERAL] = {
                .procname       = "nf_conntrack_tcp_be_liberal",
                .maxlen                = sizeof(u8),
                .mode           = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
        [NF_SYSCTL_CT_PROTO_TCP_IGNORE_INVALID_RST] = {
                .procname        = "nf_conntrack_tcp_ignore_invalid_rst",
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_ONE,
        },
        [NF_SYSCTL_CT_PROTO_TCP_MAX_RETRANS] = {
                .procname        = "nf_conntrack_tcp_max_retrans",
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_UDP] = {
                .procname        = "nf_conntrack_udp_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM] = {
                .procname        = "nf_conntrack_udp_timeout_stream",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        [NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD] = {
                .procname        = "nf_flowtable_udp_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#endif
        [NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP] = {
                .procname        = "nf_conntrack_icmp_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6] = {
                .procname        = "nf_conntrack_icmpv6_timeout",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#ifdef CONFIG_NF_CT_PROTO_SCTP
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_CLOSED] = {
                .procname        = "nf_conntrack_sctp_timeout_closed",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_WAIT] = {
                .procname        = "nf_conntrack_sctp_timeout_cookie_wait",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_COOKIE_ECHOED] = {
                .procname        = "nf_conntrack_sctp_timeout_cookie_echoed",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_ESTABLISHED] = {
                .procname        = "nf_conntrack_sctp_timeout_established",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_SENT] = {
                .procname        = "nf_conntrack_sctp_timeout_shutdown_sent",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_RECD] = {
                .procname        = "nf_conntrack_sctp_timeout_shutdown_recd",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_SHUTDOWN_ACK_SENT] = {
                .procname        = "nf_conntrack_sctp_timeout_shutdown_ack_sent",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_HEARTBEAT_SENT] = {
                .procname        = "nf_conntrack_sctp_timeout_heartbeat_sent",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
#endif
#ifdef CONFIG_NF_CT_PROTO_DCCP
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_REQUEST] = {
                .procname        = "nf_conntrack_dccp_timeout_request",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_RESPOND] = {
                .procname        = "nf_conntrack_dccp_timeout_respond",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_PARTOPEN] = {
                .procname        = "nf_conntrack_dccp_timeout_partopen",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_OPEN] = {
                .procname        = "nf_conntrack_dccp_timeout_open",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSEREQ] = {
                .procname        = "nf_conntrack_dccp_timeout_closereq",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_CLOSING] = {
                .procname        = "nf_conntrack_dccp_timeout_closing",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_TIMEWAIT] = {
                .procname        = "nf_conntrack_dccp_timeout_timewait",
                .maxlen                = sizeof(unsigned int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_DCCP_LOOSE] = {
                .procname        = "nf_conntrack_dccp_loose",
                .maxlen                = sizeof(u8),
                .mode                = 0644,
                .proc_handler        = proc_dou8vec_minmax,
                .extra1         = SYSCTL_ZERO,
                .extra2         = SYSCTL_ONE,
        },
#endif
#ifdef CONFIG_NF_CT_PROTO_GRE
        [NF_SYSCTL_CT_PROTO_TIMEOUT_GRE] = {
                .procname       = "nf_conntrack_gre_timeout",
                .maxlen         = sizeof(unsigned int),
                .mode           = 0644,
                .proc_handler   = proc_dointvec_jiffies,
        },
        [NF_SYSCTL_CT_PROTO_TIMEOUT_GRE_STREAM] = {
                .procname       = "nf_conntrack_gre_timeout_stream",
                .maxlen         = sizeof(unsigned int),
                .mode           = 0644,
                .proc_handler   = proc_dointvec_jiffies,
        },
#endif
};

static struct ctl_table nf_ct_netfilter_table[] = {
        {
                .procname        = "nf_conntrack_max",
                .data                = &nf_conntrack_max,
                .maxlen                = sizeof(int),
                .mode                = 0644,
                .proc_handler        = proc_dointvec_minmax,
                .extra1                = SYSCTL_ZERO,
                .extra2                = SYSCTL_INT_MAX,
        },
};

static void nf_conntrack_standalone_init_tcp_sysctl(struct net *net,
                                                    struct ctl_table *table)
{
        struct nf_tcp_net *tn = nf_tcp_pernet(net);

#define XASSIGN(XNAME, tn) \
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_ ## XNAME].data = \
                        &(tn)->timeouts[TCP_CONNTRACK_ ## XNAME]

        XASSIGN(SYN_SENT, tn);
        XASSIGN(SYN_RECV, tn);
        XASSIGN(ESTABLISHED, tn);
        XASSIGN(FIN_WAIT, tn);
        XASSIGN(CLOSE_WAIT, tn);
        XASSIGN(LAST_ACK, tn);
        XASSIGN(TIME_WAIT, tn);
        XASSIGN(CLOSE, tn);
        XASSIGN(RETRANS, tn);
        XASSIGN(UNACK, tn);
#undef XASSIGN
#define XASSIGN(XNAME, rval) \
        table[NF_SYSCTL_CT_PROTO_TCP_ ## XNAME].data = (rval)

        XASSIGN(LOOSE, &tn->tcp_loose);
        XASSIGN(LIBERAL, &tn->tcp_be_liberal);
        XASSIGN(MAX_RETRANS, &tn->tcp_max_retrans);
        XASSIGN(IGNORE_INVALID_RST, &tn->tcp_ignore_invalid_rst);
#undef XASSIGN

#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD].data = &tn->offload_timeout;
#endif

}

static void nf_conntrack_standalone_init_sctp_sysctl(struct net *net,
                                                     struct ctl_table *table)
{
#ifdef CONFIG_NF_CT_PROTO_SCTP
        struct nf_sctp_net *sn = nf_sctp_pernet(net);

#define XASSIGN(XNAME, sn) \
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_SCTP_ ## XNAME].data = \
                        &(sn)->timeouts[SCTP_CONNTRACK_ ## XNAME]

        XASSIGN(CLOSED, sn);
        XASSIGN(COOKIE_WAIT, sn);
        XASSIGN(COOKIE_ECHOED, sn);
        XASSIGN(ESTABLISHED, sn);
        XASSIGN(SHUTDOWN_SENT, sn);
        XASSIGN(SHUTDOWN_RECD, sn);
        XASSIGN(SHUTDOWN_ACK_SENT, sn);
        XASSIGN(HEARTBEAT_SENT, sn);
#undef XASSIGN
#endif
}

static void nf_conntrack_standalone_init_dccp_sysctl(struct net *net,
                                                     struct ctl_table *table)
{
#ifdef CONFIG_NF_CT_PROTO_DCCP
        struct nf_dccp_net *dn = nf_dccp_pernet(net);

#define XASSIGN(XNAME, dn) \
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_DCCP_ ## XNAME].data = \
                        &(dn)->dccp_timeout[CT_DCCP_ ## XNAME]

        XASSIGN(REQUEST, dn);
        XASSIGN(RESPOND, dn);
        XASSIGN(PARTOPEN, dn);
        XASSIGN(OPEN, dn);
        XASSIGN(CLOSEREQ, dn);
        XASSIGN(CLOSING, dn);
        XASSIGN(TIMEWAIT, dn);
#undef XASSIGN

        table[NF_SYSCTL_CT_PROTO_DCCP_LOOSE].data = &dn->dccp_loose;
#endif
}

static void nf_conntrack_standalone_init_gre_sysctl(struct net *net,
                                                    struct ctl_table *table)
{
#ifdef CONFIG_NF_CT_PROTO_GRE
        struct nf_gre_net *gn = nf_gre_pernet(net);

        table[NF_SYSCTL_CT_PROTO_TIMEOUT_GRE].data = &gn->timeouts[GRE_CT_UNREPLIED];
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_GRE_STREAM].data = &gn->timeouts[GRE_CT_REPLIED];
#endif
}

static int nf_conntrack_standalone_init_sysctl(struct net *net)
{
        struct nf_conntrack_net *cnet = nf_ct_pernet(net);
        struct nf_udp_net *un = nf_udp_pernet(net);
        struct ctl_table *table;

        BUILD_BUG_ON(ARRAY_SIZE(nf_ct_sysctl_table) != NF_SYSCTL_CT_LAST_SYSCTL);

        table = kmemdup(nf_ct_sysctl_table, sizeof(nf_ct_sysctl_table),
                        GFP_KERNEL);
        if (!table)
                return -ENOMEM;

        table[NF_SYSCTL_CT_COUNT].data = &cnet->count;
        table[NF_SYSCTL_CT_CHECKSUM].data = &net->ct.sysctl_checksum;
        table[NF_SYSCTL_CT_LOG_INVALID].data = &net->ct.sysctl_log_invalid;
        table[NF_SYSCTL_CT_ACCT].data = &net->ct.sysctl_acct;
#ifdef CONFIG_NF_CONNTRACK_EVENTS
        table[NF_SYSCTL_CT_EVENTS].data = &net->ct.sysctl_events;
#endif
#ifdef CONFIG_NF_CONNTRACK_TIMESTAMP
        table[NF_SYSCTL_CT_TIMESTAMP].data = &net->ct.sysctl_tstamp;
#endif
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_GENERIC].data = &nf_generic_pernet(net)->timeout;
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP].data = &nf_icmp_pernet(net)->timeout;
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6].data = &nf_icmpv6_pernet(net)->timeout;
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP].data = &un->timeouts[UDP_CT_UNREPLIED];
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM].data = &un->timeouts[UDP_CT_REPLIED];
#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
        table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD].data = &un->offload_timeout;
#endif

        nf_conntrack_standalone_init_tcp_sysctl(net, table);
        nf_conntrack_standalone_init_sctp_sysctl(net, table);
        nf_conntrack_standalone_init_dccp_sysctl(net, table);
        nf_conntrack_standalone_init_gre_sysctl(net, table);

        /* Don't allow non-init_net ns to alter global sysctls */
        if (!net_eq(&init_net, net)) {
                table[NF_SYSCTL_CT_MAX].mode = 0444;
                table[NF_SYSCTL_CT_EXPECT_MAX].mode = 0444;
                table[NF_SYSCTL_CT_BUCKETS].mode = 0444;
        }

        cnet->sysctl_header = register_net_sysctl_sz(net, "net/netfilter",
                                                     table,
                                                     ARRAY_SIZE(nf_ct_sysctl_table));
        if (!cnet->sysctl_header)
                goto out_unregister_netfilter;

        return 0;

out_unregister_netfilter:
        kfree(table);
        return -ENOMEM;
}

static void nf_conntrack_standalone_fini_sysctl(struct net *net)
{
        struct nf_conntrack_net *cnet = nf_ct_pernet(net);
        const struct ctl_table *table;

        table = cnet->sysctl_header->ctl_table_arg;
        unregister_net_sysctl_table(cnet->sysctl_header);
        kfree(table);
}
#else
static int nf_conntrack_standalone_init_sysctl(struct net *net)
{
        return 0;
}

static void nf_conntrack_standalone_fini_sysctl(struct net *net)
{
}
#endif /* CONFIG_SYSCTL */

static void nf_conntrack_fini_net(struct net *net)
{
        if (enable_hooks)
                nf_ct_netns_put(net, NFPROTO_INET);

        nf_conntrack_standalone_fini_proc(net);
        nf_conntrack_standalone_fini_sysctl(net);
}

static int nf_conntrack_pernet_init(struct net *net)
{
        int ret;

        net->ct.sysctl_checksum = 1;

        ret = nf_conntrack_standalone_init_sysctl(net);
        if (ret < 0)
                return ret;

        ret = nf_conntrack_standalone_init_proc(net);
        if (ret < 0)
                goto out_proc;

        ret = nf_conntrack_init_net(net);
        if (ret < 0)
                goto out_init_net;

        if (enable_hooks) {
                ret = nf_ct_netns_get(net, NFPROTO_INET);
                if (ret < 0)
                        goto out_hooks;
        }

        return 0;

out_hooks:
        nf_conntrack_cleanup_net(net);
out_init_net:
        nf_conntrack_standalone_fini_proc(net);
out_proc:
        nf_conntrack_standalone_fini_sysctl(net);
        return ret;
}

static void nf_conntrack_pernet_exit(struct list_head *net_exit_list)
{
        struct net *net;

        list_for_each_entry(net, net_exit_list, exit_list)
                nf_conntrack_fini_net(net);

        nf_conntrack_cleanup_net_list(net_exit_list);
}

static struct pernet_operations nf_conntrack_net_ops = {
        .init                = nf_conntrack_pernet_init,
        .exit_batch        = nf_conntrack_pernet_exit,
        .id                = &nf_conntrack_net_id,
        .size = sizeof(struct nf_conntrack_net),
};

static int __init nf_conntrack_standalone_init(void)
{
        int ret = nf_conntrack_init_start();
        if (ret < 0)
                goto out_start;

        BUILD_BUG_ON(NFCT_INFOMASK <= IP_CT_NUMBER);

#ifdef CONFIG_SYSCTL
        nf_ct_netfilter_header =
                register_net_sysctl(&init_net, "net", nf_ct_netfilter_table);
        if (!nf_ct_netfilter_header) {
                pr_err("nf_conntrack: can't register to sysctl.\n");
                ret = -ENOMEM;
                goto out_sysctl;
        }

        nf_conntrack_htable_size_user = nf_conntrack_htable_size;
#endif

        nf_conntrack_init_end();

        ret = register_pernet_subsys(&nf_conntrack_net_ops);
        if (ret < 0)
                goto out_pernet;

        return 0;

out_pernet:
#ifdef CONFIG_SYSCTL
        unregister_net_sysctl_table(nf_ct_netfilter_header);
out_sysctl:
#endif
        nf_conntrack_cleanup_end();
out_start:
        return ret;
}

static void __exit nf_conntrack_standalone_fini(void)
{
        nf_conntrack_cleanup_start();
        unregister_pernet_subsys(&nf_conntrack_net_ops);
#ifdef CONFIG_SYSCTL
        unregister_net_sysctl_table(nf_ct_netfilter_header);
#endif
        nf_conntrack_cleanup_end();
}

module_init(nf_conntrack_standalone_init);
module_exit(nf_conntrack_standalone_fini);


















































































































































   24 




    4 
























































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_BYTEORDER_GENERIC_H
#define _LINUX_BYTEORDER_GENERIC_H

/*
 * linux/byteorder/generic.h
 * Generic Byte-reordering support
 *
 * The "... p" macros, like le64_to_cpup, can be used with pointers
 * to unaligned data, but there will be a performance penalty on 
 * some architectures.  Use get_unaligned for unaligned data.
 *
 * Francois-Rene Rideau <fare@tunes.org> 19970707
 *    gathered all the good ideas from all asm-foo/byteorder.h into one file,
 *    cleaned them up.
 *    I hope it is compliant with non-GCC compilers.
 *    I decided to put __BYTEORDER_HAS_U64__ in byteorder.h,
 *    because I wasn't sure it would be ok to put it in types.h
 *    Upgraded it to 2.1.43
 * Francois-Rene Rideau <fare@tunes.org> 19971012
 *    Upgraded it to 2.1.57
 *    to please Linus T., replaced huge #ifdef's between little/big endian
 *    by nestedly #include'd files.
 * Francois-Rene Rideau <fare@tunes.org> 19971205
 *    Made it to 2.1.71; now a facelift:
 *    Put files under include/linux/byteorder/
 *    Split swab from generic support.
 *
 * TODO:
 *   = Regular kernel maintainers could also replace all these manual
 *    byteswap macros that remain, disseminated among drivers,
 *    after some grep or the sources...
 *   = Linus might want to rename all these macros and files to fit his taste,
 *    to fit his personal naming scheme.
 *   = it seems that a few drivers would also appreciate
 *    nybble swapping support...
 *   = every architecture could add their byteswap macro in asm/byteorder.h
 *    see how some architectures already do (i386, alpha, ppc, etc)
 *   = cpu_to_beXX and beXX_to_cpu might some day need to be well
 *    distinguished throughout the kernel. This is not the case currently,
 *    since little endian, big endian, and pdp endian machines needn't it.
 *    But this might be the case for, say, a port of Linux to 20/21 bit
 *    architectures (and F21 Linux addict around?).
 */

/*
 * The following macros are to be defined by <asm/byteorder.h>:
 *
 * Conversion of long and short int between network and host format
 *        ntohl(__u32 x)
 *        ntohs(__u16 x)
 *        htonl(__u32 x)
 *        htons(__u16 x)
 * It seems that some programs (which? where? or perhaps a standard? POSIX?)
 * might like the above to be functions, not macros (why?).
 * if that's true, then detect them, and take measures.
 * Anyway, the measure is: define only ___ntohl as a macro instead,
 * and in a separate file, have
 * unsigned long inline ntohl(x){return ___ntohl(x);}
 *
 * The same for constant arguments
 *        __constant_ntohl(__u32 x)
 *        __constant_ntohs(__u16 x)
 *        __constant_htonl(__u32 x)
 *        __constant_htons(__u16 x)
 *
 * Conversion of XX-bit integers (16- 32- or 64-)
 * between native CPU format and little/big endian format
 * 64-bit stuff only defined for proper architectures
 *        cpu_to_[bl]eXX(__uXX x)
 *        [bl]eXX_to_cpu(__uXX x)
 *
 * The same, but takes a pointer to the value to convert
 *        cpu_to_[bl]eXXp(__uXX x)
 *        [bl]eXX_to_cpup(__uXX x)
 *
 * The same, but change in situ
 *        cpu_to_[bl]eXXs(__uXX x)
 *        [bl]eXX_to_cpus(__uXX x)
 *
 * See asm-foo/byteorder.h for examples of how to provide
 * architecture-optimized versions
 *
 */

#define cpu_to_le64 __cpu_to_le64
#define le64_to_cpu __le64_to_cpu
#define cpu_to_le32 __cpu_to_le32
#define le32_to_cpu __le32_to_cpu
#define cpu_to_le16 __cpu_to_le16
#define le16_to_cpu __le16_to_cpu
#define cpu_to_be64 __cpu_to_be64
#define be64_to_cpu __be64_to_cpu
#define cpu_to_be32 __cpu_to_be32
#define be32_to_cpu __be32_to_cpu
#define cpu_to_be16 __cpu_to_be16
#define be16_to_cpu __be16_to_cpu
#define cpu_to_le64p __cpu_to_le64p
#define le64_to_cpup __le64_to_cpup
#define cpu_to_le32p __cpu_to_le32p
#define le32_to_cpup __le32_to_cpup
#define cpu_to_le16p __cpu_to_le16p
#define le16_to_cpup __le16_to_cpup
#define cpu_to_be64p __cpu_to_be64p
#define be64_to_cpup __be64_to_cpup
#define cpu_to_be32p __cpu_to_be32p
#define be32_to_cpup __be32_to_cpup
#define cpu_to_be16p __cpu_to_be16p
#define be16_to_cpup __be16_to_cpup
#define cpu_to_le64s __cpu_to_le64s
#define le64_to_cpus __le64_to_cpus
#define cpu_to_le32s __cpu_to_le32s
#define le32_to_cpus __le32_to_cpus
#define cpu_to_le16s __cpu_to_le16s
#define le16_to_cpus __le16_to_cpus
#define cpu_to_be64s __cpu_to_be64s
#define be64_to_cpus __be64_to_cpus
#define cpu_to_be32s __cpu_to_be32s
#define be32_to_cpus __be32_to_cpus
#define cpu_to_be16s __cpu_to_be16s
#define be16_to_cpus __be16_to_cpus

/*
 * They have to be macros in order to do the constant folding
 * correctly - if the argument passed into a inline function
 * it is no longer constant according to gcc..
 */

#undef ntohl
#undef ntohs
#undef htonl
#undef htons

#define ___htonl(x) __cpu_to_be32(x)
#define ___htons(x) __cpu_to_be16(x)
#define ___ntohl(x) __be32_to_cpu(x)
#define ___ntohs(x) __be16_to_cpu(x)

#define htonl(x) ___htonl(x)
#define ntohl(x) ___ntohl(x)
#define htons(x) ___htons(x)
#define ntohs(x) ___ntohs(x)

static inline void le16_add_cpu(__le16 *var, u16 val)
{
        *var = cpu_to_le16(le16_to_cpu(*var) + val);
}

static inline void le32_add_cpu(__le32 *var, u32 val)
{
        *var = cpu_to_le32(le32_to_cpu(*var) + val);
}

static inline void le64_add_cpu(__le64 *var, u64 val)
{
        *var = cpu_to_le64(le64_to_cpu(*var) + val);
}

/* XXX: this stuff can be optimized */
static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
{
        while (words--) {
                __le32_to_cpus(buf);
                buf++;
        }
}

static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
{
        while (words--) {
                __cpu_to_le32s(buf);
                buf++;
        }
}

static inline void be16_add_cpu(__be16 *var, u16 val)
{
        *var = cpu_to_be16(be16_to_cpu(*var) + val);
}

static inline void be32_add_cpu(__be32 *var, u32 val)
{
        *var = cpu_to_be32(be32_to_cpu(*var) + val);
}

static inline void be64_add_cpu(__be64 *var, u64 val)
{
        *var = cpu_to_be64(be64_to_cpu(*var) + val);
}

static inline void cpu_to_be32_array(__be32 *dst, const u32 *src, size_t len)
{
        size_t i;

        for (i = 0; i < len; i++)
                dst[i] = cpu_to_be32(src[i]);
}

static inline void be32_to_cpu_array(u32 *dst, const __be32 *src, size_t len)
{
        size_t i;

        for (i = 0; i < len; i++)
                dst[i] = be32_to_cpu(src[i]);
}

#endif /* _LINUX_BYTEORDER_GENERIC_H */



















































    4 

















































    7 






















    1 




    1 



















    4 


    4 








    4 







    4 
















    7 


    7 

    2 







    7 
    1 

    1 




    7 


































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * NetLabel Network Address Lists
 *
 * This file contains network address list functions used to manage ordered
 * lists of network addresses for use by the NetLabel subsystem.  The NetLabel
 * system manages static and dynamic label mappings for network protocols such
 * as CIPSO and RIPSO.
 *
 * Author: Paul Moore <paul@paul-moore.com>
 */

/*
 * (c) Copyright Hewlett-Packard Development Company, L.P., 2008
 */

#include <linux/types.h>
#include <linux/rcupdate.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <net/ip.h>
#include <net/ipv6.h>
#include <linux/audit.h>

#include "netlabel_addrlist.h"

/*
 * Address List Functions
 */

/**
 * netlbl_af4list_search - Search for a matching IPv4 address entry
 * @addr: IPv4 address
 * @head: the list head
 *
 * Description:
 * Searches the IPv4 address list given by @head.  If a matching address entry
 * is found it is returned, otherwise NULL is returned.  The caller is
 * responsible for calling the rcu_read_[un]lock() functions.
 *
 */
struct netlbl_af4list *netlbl_af4list_search(__be32 addr,
                                             struct list_head *head)
{
        struct netlbl_af4list *iter;

        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid && (addr & iter->mask) == iter->addr)
                        return iter;

        return NULL;
}

/**
 * netlbl_af4list_search_exact - Search for an exact IPv4 address entry
 * @addr: IPv4 address
 * @mask: IPv4 address mask
 * @head: the list head
 *
 * Description:
 * Searches the IPv4 address list given by @head.  If an exact match if found
 * it is returned, otherwise NULL is returned.  The caller is responsible for
 * calling the rcu_read_[un]lock() functions.
 *
 */
struct netlbl_af4list *netlbl_af4list_search_exact(__be32 addr,
                                                   __be32 mask,
                                                   struct list_head *head)
{
        struct netlbl_af4list *iter;

        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid && iter->addr == addr && iter->mask == mask)
                        return iter;

        return NULL;
}


#if IS_ENABLED(CONFIG_IPV6)
/**
 * netlbl_af6list_search - Search for a matching IPv6 address entry
 * @addr: IPv6 address
 * @head: the list head
 *
 * Description:
 * Searches the IPv6 address list given by @head.  If a matching address entry
 * is found it is returned, otherwise NULL is returned.  The caller is
 * responsible for calling the rcu_read_[un]lock() functions.
 *
 */
struct netlbl_af6list *netlbl_af6list_search(const struct in6_addr *addr,
                                             struct list_head *head)
{
        struct netlbl_af6list *iter;

        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid &&
                    ipv6_masked_addr_cmp(&iter->addr, &iter->mask, addr) == 0)
                        return iter;

        return NULL;
}

/**
 * netlbl_af6list_search_exact - Search for an exact IPv6 address entry
 * @addr: IPv6 address
 * @mask: IPv6 address mask
 * @head: the list head
 *
 * Description:
 * Searches the IPv6 address list given by @head.  If an exact match if found
 * it is returned, otherwise NULL is returned.  The caller is responsible for
 * calling the rcu_read_[un]lock() functions.
 *
 */
struct netlbl_af6list *netlbl_af6list_search_exact(const struct in6_addr *addr,
                                                   const struct in6_addr *mask,
                                                   struct list_head *head)
{
        struct netlbl_af6list *iter;

        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid &&
                    ipv6_addr_equal(&iter->addr, addr) &&
                    ipv6_addr_equal(&iter->mask, mask))
                        return iter;

        return NULL;
}
#endif /* IPv6 */

/**
 * netlbl_af4list_add - Add a new IPv4 address entry to a list
 * @entry: address entry
 * @head: the list head
 *
 * Description:
 * Add a new address entry to the list pointed to by @head.  On success zero is
 * returned, otherwise a negative value is returned.  The caller is responsible
 * for calling the necessary locking functions.
 *
 */
int netlbl_af4list_add(struct netlbl_af4list *entry, struct list_head *head)
{
        struct netlbl_af4list *iter;

        iter = netlbl_af4list_search(entry->addr, head);
        if (iter != NULL &&
            iter->addr == entry->addr && iter->mask == entry->mask)
                return -EEXIST;

        /* in order to speed up address searches through the list (the common
         * case) we need to keep the list in order based on the size of the
         * address mask such that the entry with the widest mask (smallest
         * numerical value) appears first in the list */
        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid &&
                    ntohl(entry->mask) > ntohl(iter->mask)) {
                        __list_add_rcu(&entry->list,
                                       iter->list.prev,
                                       &iter->list);
                        return 0;
                }
        list_add_tail_rcu(&entry->list, head);
        return 0;
}

#if IS_ENABLED(CONFIG_IPV6)
/**
 * netlbl_af6list_add - Add a new IPv6 address entry to a list
 * @entry: address entry
 * @head: the list head
 *
 * Description:
 * Add a new address entry to the list pointed to by @head.  On success zero is
 * returned, otherwise a negative value is returned.  The caller is responsible
 * for calling the necessary locking functions.
 *
 */
int netlbl_af6list_add(struct netlbl_af6list *entry, struct list_head *head)
{
        struct netlbl_af6list *iter;

        iter = netlbl_af6list_search(&entry->addr, head);
        if (iter != NULL &&
            ipv6_addr_equal(&iter->addr, &entry->addr) &&
            ipv6_addr_equal(&iter->mask, &entry->mask))
                return -EEXIST;

        /* in order to speed up address searches through the list (the common
         * case) we need to keep the list in order based on the size of the
         * address mask such that the entry with the widest mask (smallest
         * numerical value) appears first in the list */
        list_for_each_entry_rcu(iter, head, list)
                if (iter->valid &&
                    ipv6_addr_cmp(&entry->mask, &iter->mask) > 0) {
                        __list_add_rcu(&entry->list,
                                       iter->list.prev,
                                       &iter->list);
                        return 0;
                }
        list_add_tail_rcu(&entry->list, head);
        return 0;
}
#endif /* IPv6 */

/**
 * netlbl_af4list_remove_entry - Remove an IPv4 address entry
 * @entry: address entry
 *
 * Description:
 * Remove the specified IP address entry.  The caller is responsible for
 * calling the necessary locking functions.
 *
 */
void netlbl_af4list_remove_entry(struct netlbl_af4list *entry)
{
        entry->valid = 0;
        list_del_rcu(&entry->list);
}

/**
 * netlbl_af4list_remove - Remove an IPv4 address entry
 * @addr: IP address
 * @mask: IP address mask
 * @head: the list head
 *
 * Description:
 * Remove an IP address entry from the list pointed to by @head.  Returns the
 * entry on success, NULL on failure.  The caller is responsible for calling
 * the necessary locking functions.
 *
 */
struct netlbl_af4list *netlbl_af4list_remove(__be32 addr, __be32 mask,
                                             struct list_head *head)
{
        struct netlbl_af4list *entry;

        entry = netlbl_af4list_search_exact(addr, mask, head);
        if (entry == NULL)
                return NULL;
        netlbl_af4list_remove_entry(entry);
        return entry;
}

#if IS_ENABLED(CONFIG_IPV6)
/**
 * netlbl_af6list_remove_entry - Remove an IPv6 address entry
 * @entry: address entry
 *
 * Description:
 * Remove the specified IP address entry.  The caller is responsible for
 * calling the necessary locking functions.
 *
 */
void netlbl_af6list_remove_entry(struct netlbl_af6list *entry)
{
        entry->valid = 0;
        list_del_rcu(&entry->list);
}

/**
 * netlbl_af6list_remove - Remove an IPv6 address entry
 * @addr: IP address
 * @mask: IP address mask
 * @head: the list head
 *
 * Description:
 * Remove an IP address entry from the list pointed to by @head.  Returns the
 * entry on success, NULL on failure.  The caller is responsible for calling
 * the necessary locking functions.
 *
 */
struct netlbl_af6list *netlbl_af6list_remove(const struct in6_addr *addr,
                                             const struct in6_addr *mask,
                                             struct list_head *head)
{
        struct netlbl_af6list *entry;

        entry = netlbl_af6list_search_exact(addr, mask, head);
        if (entry == NULL)
                return NULL;
        netlbl_af6list_remove_entry(entry);
        return entry;
}
#endif /* IPv6 */

/*
 * Audit Helper Functions
 */

#ifdef CONFIG_AUDIT
/**
 * netlbl_af4list_audit_addr - Audit an IPv4 address
 * @audit_buf: audit buffer
 * @src: true if source address, false if destination
 * @dev: network interface
 * @addr: IP address
 * @mask: IP address mask
 *
 * Description:
 * Write the IPv4 address and address mask, if necessary, to @audit_buf.
 *
 */
void netlbl_af4list_audit_addr(struct audit_buffer *audit_buf,
                                        int src, const char *dev,
                                        __be32 addr, __be32 mask)
{
        u32 mask_val = ntohl(mask);
        char *dir = (src ? "src" : "dst");

        if (dev != NULL)
                audit_log_format(audit_buf, " netif=%s", dev);
        audit_log_format(audit_buf, " %s=%pI4", dir, &addr);
        if (mask_val != 0xffffffff) {
                u32 mask_len = 0;
                while (mask_val > 0) {
                        mask_val <<= 1;
                        mask_len++;
                }
                audit_log_format(audit_buf, " %s_prefixlen=%d", dir, mask_len);
        }
}

#if IS_ENABLED(CONFIG_IPV6)
/**
 * netlbl_af6list_audit_addr - Audit an IPv6 address
 * @audit_buf: audit buffer
 * @src: true if source address, false if destination
 * @dev: network interface
 * @addr: IP address
 * @mask: IP address mask
 *
 * Description:
 * Write the IPv6 address and address mask, if necessary, to @audit_buf.
 *
 */
void netlbl_af6list_audit_addr(struct audit_buffer *audit_buf,
                                 int src,
                                 const char *dev,
                                 const struct in6_addr *addr,
                                 const struct in6_addr *mask)
{
        char *dir = (src ? "src" : "dst");

        if (dev != NULL)
                audit_log_format(audit_buf, " netif=%s", dev);
        audit_log_format(audit_buf, " %s=%pI6", dir, addr);
        if (ntohl(mask->s6_addr32[3]) != 0xffffffff) {
                u32 mask_len = 0;
                u32 mask_val;
                int iter = -1;
                while (ntohl(mask->s6_addr32[++iter]) == 0xffffffff)
                        mask_len += 32;
                mask_val = ntohl(mask->s6_addr32[iter]);
                while (mask_val > 0) {
                        mask_val <<= 1;
                        mask_len++;
                }
                audit_log_format(audit_buf, " %s_prefixlen=%d", dir, mask_len);
        }
}
#endif /* IPv6 */
#endif /* CONFIG_AUDIT */


























































    8 




    8 






    8 
    8 
































    8 






























































































































































































































































































































































































































































































































    8 


















































































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
// SPDX-License-Identifier: GPL-2.0
/*
 * This file contains functions which emulate a local clock-event
 * device via a broadcast event source.
 *
 * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
 * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
 * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
 */
#include <linux/cpu.h>
#include <linux/err.h>
#include <linux/hrtimer.h>
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/profile.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/module.h>

#include "tick-internal.h"

/*
 * Broadcast support for broken x86 hardware, where the local apic
 * timer stops in C3 state.
 */

static struct tick_device tick_broadcast_device;
static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
static cpumask_var_t tmpmask __cpumask_var_read_mostly;
static int tick_broadcast_forced;

static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);

#ifdef CONFIG_TICK_ONESHOT
static DEFINE_PER_CPU(struct clock_event_device *, tick_oneshot_wakeup_device);

static void tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic);
static void tick_broadcast_clear_oneshot(int cpu);
static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
# ifdef CONFIG_HOTPLUG_CPU
static void tick_broadcast_oneshot_offline(unsigned int cpu);
# endif
#else
static inline void
tick_broadcast_setup_oneshot(struct clock_event_device *bc, bool from_periodic) { BUG(); }
static inline void tick_broadcast_clear_oneshot(int cpu) { }
static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
# ifdef CONFIG_HOTPLUG_CPU
static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }
# endif
#endif

/*
 * Debugging: see timer_list.c
 */
struct tick_device *tick_get_broadcast_device(void)
{
        return &tick_broadcast_device;
}

struct cpumask *tick_get_broadcast_mask(void)
{
        return tick_broadcast_mask;
}

static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu);

const struct clock_event_device *tick_get_wakeup_device(int cpu)
{
        return tick_get_oneshot_wakeup_device(cpu);
}

/*
 * Start the device in periodic mode
 */
static void tick_broadcast_start_periodic(struct clock_event_device *bc)
{
        if (bc)
                tick_setup_periodic(bc, 1);
}

/*
 * Check, if the device can be utilized as broadcast device:
 */
static bool tick_check_broadcast_device(struct clock_event_device *curdev,
                                        struct clock_event_device *newdev)
{
        if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
            (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
            (newdev->features & CLOCK_EVT_FEAT_C3STOP))
                return false;

        if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
            !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
                return false;

        return !curdev || newdev->rating > curdev->rating;
}

#ifdef CONFIG_TICK_ONESHOT
static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu)
{
        return per_cpu(tick_oneshot_wakeup_device, cpu);
}

static void tick_oneshot_wakeup_handler(struct clock_event_device *wd)
{
        /*
         * If we woke up early and the tick was reprogrammed in the
         * meantime then this may be spurious but harmless.
         */
        tick_receive_broadcast();
}

static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,
                                           int cpu)
{
        struct clock_event_device *curdev = tick_get_oneshot_wakeup_device(cpu);

        if (!newdev)
                goto set_device;

        if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
            (newdev->features & CLOCK_EVT_FEAT_C3STOP))
                 return false;

        if (!(newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
            !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
                return false;

        if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
                return false;

        if (curdev && newdev->rating <= curdev->rating)
                return false;

        if (!try_module_get(newdev->owner))
                return false;

        newdev->event_handler = tick_oneshot_wakeup_handler;
set_device:
        clockevents_exchange_device(curdev, newdev);
        per_cpu(tick_oneshot_wakeup_device, cpu) = newdev;
        return true;
}
#else
static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu)
{
        return NULL;
}

static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev,
                                           int cpu)
{
        return false;
}
#endif

/*
 * Conditionally install/replace broadcast device
 */
void tick_install_broadcast_device(struct clock_event_device *dev, int cpu)
{
        struct clock_event_device *cur = tick_broadcast_device.evtdev;

        if (tick_set_oneshot_wakeup_device(dev, cpu))
                return;

        if (!tick_check_broadcast_device(cur, dev))
                return;

        if (!try_module_get(dev->owner))
                return;

        clockevents_exchange_device(cur, dev);
        if (cur)
                cur->event_handler = clockevents_handle_noop;
        tick_broadcast_device.evtdev = dev;
        if (!cpumask_empty(tick_broadcast_mask))
                tick_broadcast_start_periodic(dev);

        if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT))
                return;

        /*
         * If the system already runs in oneshot mode, switch the newly
         * registered broadcast device to oneshot mode explicitly.
         */
        if (tick_broadcast_oneshot_active()) {
                tick_broadcast_switch_to_oneshot();
                return;
        }

        /*
         * Inform all cpus about this. We might be in a situation
         * where we did not switch to oneshot mode because the per cpu
         * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
         * of a oneshot capable broadcast device. Without that
         * notification the systems stays stuck in periodic mode
         * forever.
         */
        tick_clock_notify();
}

/*
 * Check, if the device is the broadcast device
 */
int tick_is_broadcast_device(struct clock_event_device *dev)
{
        return (dev && tick_broadcast_device.evtdev == dev);
}

int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
{
        int ret = -ENODEV;

        if (tick_is_broadcast_device(dev)) {
                raw_spin_lock(&tick_broadcast_lock);
                ret = __clockevents_update_freq(dev, freq);
                raw_spin_unlock(&tick_broadcast_lock);
        }
        return ret;
}


static void err_broadcast(const struct cpumask *mask)
{
        pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
}

static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
{
        if (!dev->broadcast)
                dev->broadcast = tick_broadcast;
        if (!dev->broadcast) {
                pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
                             dev->name);
                dev->broadcast = err_broadcast;
        }
}

/*
 * Check, if the device is dysfunctional and a placeholder, which
 * needs to be handled by the broadcast device.
 */
int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
{
        struct clock_event_device *bc = tick_broadcast_device.evtdev;
        unsigned long flags;
        int ret = 0;

        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

        /*
         * Devices might be registered with both periodic and oneshot
         * mode disabled. This signals, that the device needs to be
         * operated from the broadcast device and is a placeholder for
         * the cpu local device.
         */
        if (!tick_device_is_functional(dev)) {
                dev->event_handler = tick_handle_periodic;
                tick_device_setup_broadcast_func(dev);
                cpumask_set_cpu(cpu, tick_broadcast_mask);
                if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
                        tick_broadcast_start_periodic(bc);
                else
                        tick_broadcast_setup_oneshot(bc, false);
                ret = 1;
        } else {
                /*
                 * Clear the broadcast bit for this cpu if the
                 * device is not power state affected.
                 */
                if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
                        cpumask_clear_cpu(cpu, tick_broadcast_mask);
                else
                        tick_device_setup_broadcast_func(dev);

                /*
                 * Clear the broadcast bit if the CPU is not in
                 * periodic broadcast on state.
                 */
                if (!cpumask_test_cpu(cpu, tick_broadcast_on))
                        cpumask_clear_cpu(cpu, tick_broadcast_mask);

                switch (tick_broadcast_device.mode) {
                case TICKDEV_MODE_ONESHOT:
                        /*
                         * If the system is in oneshot mode we can
                         * unconditionally clear the oneshot mask bit,
                         * because the CPU is running and therefore
                         * not in an idle state which causes the power
                         * state affected device to stop. Let the
                         * caller initialize the device.
                         */
                        tick_broadcast_clear_oneshot(cpu);
                        ret = 0;
                        break;

                case TICKDEV_MODE_PERIODIC:
                        /*
                         * If the system is in periodic mode, check
                         * whether the broadcast device can be
                         * switched off now.
                         */
                        if (cpumask_empty(tick_broadcast_mask) && bc)
                                clockevents_shutdown(bc);
                        /*
                         * If we kept the cpu in the broadcast mask,
                         * tell the caller to leave the per cpu device
                         * in shutdown state. The periodic interrupt
                         * is delivered by the broadcast device, if
                         * the broadcast device exists and is not
                         * hrtimer based.
                         */
                        if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
                                ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
                        break;
                default:
                        break;
                }
        }
        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
        return ret;
}

int tick_receive_broadcast(void)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        struct clock_event_device *evt = td->evtdev;

        if (!evt)
                return -ENODEV;

        if (!evt->event_handler)
                return -EINVAL;

        evt->event_handler(evt);
        return 0;
}

/*
 * Broadcast the event to the cpus, which are set in the mask (mangled).
 */
static bool tick_do_broadcast(struct cpumask *mask)
{
        int cpu = smp_processor_id();
        struct tick_device *td;
        bool local = false;

        /*
         * Check, if the current cpu is in the mask
         */
        if (cpumask_test_cpu(cpu, mask)) {
                struct clock_event_device *bc = tick_broadcast_device.evtdev;

                cpumask_clear_cpu(cpu, mask);
                /*
                 * We only run the local handler, if the broadcast
                 * device is not hrtimer based. Otherwise we run into
                 * a hrtimer recursion.
                 *
                 * local timer_interrupt()
                 *   local_handler()
                 *     expire_hrtimers()
                 *       bc_handler()
                 *         local_handler()
                 *             expire_hrtimers()
                 */
                local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
        }

        if (!cpumask_empty(mask)) {
                /*
                 * It might be necessary to actually check whether the devices
                 * have different broadcast functions. For now, just use the
                 * one of the first device. This works as long as we have this
                 * misfeature only on x86 (lapic)
                 */
                td = &per_cpu(tick_cpu_device, cpumask_first(mask));
                td->evtdev->broadcast(mask);
        }
        return local;
}

/*
 * Periodic broadcast:
 * - invoke the broadcast handlers
 */
static bool tick_do_periodic_broadcast(void)
{
        cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
        return tick_do_broadcast(tmpmask);
}

/*
 * Event handler for periodic broadcast ticks
 */
static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        bool bc_local;

        raw_spin_lock(&tick_broadcast_lock);

        /* Handle spurious interrupts gracefully */
        if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
                raw_spin_unlock(&tick_broadcast_lock);
                return;
        }

        bc_local = tick_do_periodic_broadcast();

        if (clockevent_state_oneshot(dev)) {
                ktime_t next = ktime_add_ns(dev->next_event, TICK_NSEC);

                clockevents_program_event(dev, next, true);
        }
        raw_spin_unlock(&tick_broadcast_lock);

        /*
         * We run the handler of the local cpu after dropping
         * tick_broadcast_lock because the handler might deadlock when
         * trying to switch to oneshot mode.
         */
        if (bc_local)
                td->evtdev->event_handler(td->evtdev);
}

/**
 * tick_broadcast_control - Enable/disable or force broadcast mode
 * @mode:        The selected broadcast mode
 *
 * Called when the system enters a state where affected tick devices
 * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
 */
void tick_broadcast_control(enum tick_broadcast_mode mode)
{
        struct clock_event_device *bc, *dev;
        struct tick_device *td;
        int cpu, bc_stopped;
        unsigned long flags;

        /* Protects also the local clockevent device. */
        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
        td = this_cpu_ptr(&tick_cpu_device);
        dev = td->evtdev;

        /*
         * Is the device not affected by the powerstate ?
         */
        if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
                goto out;

        if (!tick_device_is_functional(dev))
                goto out;

        cpu = smp_processor_id();
        bc = tick_broadcast_device.evtdev;
        bc_stopped = cpumask_empty(tick_broadcast_mask);

        switch (mode) {
        case TICK_BROADCAST_FORCE:
                tick_broadcast_forced = 1;
                fallthrough;
        case TICK_BROADCAST_ON:
                cpumask_set_cpu(cpu, tick_broadcast_on);
                if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
                        /*
                         * Only shutdown the cpu local device, if:
                         *
                         * - the broadcast device exists
                         * - the broadcast device is not a hrtimer based one
                         * - the broadcast device is in periodic mode to
                         *   avoid a hiccup during switch to oneshot mode
                         */
                        if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
                            tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
                                clockevents_shutdown(dev);
                }
                break;

        case TICK_BROADCAST_OFF:
                if (tick_broadcast_forced)
                        break;
                cpumask_clear_cpu(cpu, tick_broadcast_on);
                if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
                        if (tick_broadcast_device.mode ==
                            TICKDEV_MODE_PERIODIC)
                                tick_setup_periodic(dev, 0);
                }
                break;
        }

        if (bc) {
                if (cpumask_empty(tick_broadcast_mask)) {
                        if (!bc_stopped)
                                clockevents_shutdown(bc);
                } else if (bc_stopped) {
                        if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
                                tick_broadcast_start_periodic(bc);
                        else
                                tick_broadcast_setup_oneshot(bc, false);
                }
        }
out:
        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}
EXPORT_SYMBOL_GPL(tick_broadcast_control);

/*
 * Set the periodic handler depending on broadcast on/off
 */
void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
{
        if (!broadcast)
                dev->event_handler = tick_handle_periodic;
        else
                dev->event_handler = tick_handle_periodic_broadcast;
}

#ifdef CONFIG_HOTPLUG_CPU
static void tick_shutdown_broadcast(void)
{
        struct clock_event_device *bc = tick_broadcast_device.evtdev;

        if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
                if (bc && cpumask_empty(tick_broadcast_mask))
                        clockevents_shutdown(bc);
        }
}

/*
 * Remove a CPU from broadcasting
 */
void tick_broadcast_offline(unsigned int cpu)
{
        raw_spin_lock(&tick_broadcast_lock);
        cpumask_clear_cpu(cpu, tick_broadcast_mask);
        cpumask_clear_cpu(cpu, tick_broadcast_on);
        tick_broadcast_oneshot_offline(cpu);
        tick_shutdown_broadcast();
        raw_spin_unlock(&tick_broadcast_lock);
}

#endif

void tick_suspend_broadcast(void)
{
        struct clock_event_device *bc;
        unsigned long flags;

        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

        bc = tick_broadcast_device.evtdev;
        if (bc)
                clockevents_shutdown(bc);

        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * This is called from tick_resume_local() on a resuming CPU. That's
 * called from the core resume function, tick_unfreeze() and the magic XEN
 * resume hackery.
 *
 * In none of these cases the broadcast device mode can change and the
 * bit of the resuming CPU in the broadcast mask is safe as well.
 */
bool tick_resume_check_broadcast(void)
{
        if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
                return false;
        else
                return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
}

void tick_resume_broadcast(void)
{
        struct clock_event_device *bc;
        unsigned long flags;

        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

        bc = tick_broadcast_device.evtdev;

        if (bc) {
                clockevents_tick_resume(bc);

                switch (tick_broadcast_device.mode) {
                case TICKDEV_MODE_PERIODIC:
                        if (!cpumask_empty(tick_broadcast_mask))
                                tick_broadcast_start_periodic(bc);
                        break;
                case TICKDEV_MODE_ONESHOT:
                        if (!cpumask_empty(tick_broadcast_mask))
                                tick_resume_broadcast_oneshot(bc);
                        break;
                }
        }
        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

#ifdef CONFIG_TICK_ONESHOT

static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;

/*
 * Exposed for debugging: see timer_list.c
 */
struct cpumask *tick_get_broadcast_oneshot_mask(void)
{
        return tick_broadcast_oneshot_mask;
}

/*
 * Called before going idle with interrupts disabled. Checks whether a
 * broadcast event from the other core is about to happen. We detected
 * that in tick_broadcast_oneshot_control(). The callsite can use this
 * to avoid a deep idle transition as we are about to get the
 * broadcast IPI right away.
 */
noinstr int tick_check_broadcast_expired(void)
{
#ifdef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
        return arch_test_bit(smp_processor_id(), cpumask_bits(tick_broadcast_force_mask));
#else
        return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
#endif
}

/*
 * Set broadcast interrupt affinity
 */
static void tick_broadcast_set_affinity(struct clock_event_device *bc,
                                        const struct cpumask *cpumask)
{
        if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
                return;

        if (cpumask_equal(bc->cpumask, cpumask))
                return;

        bc->cpumask = cpumask;
        irq_set_affinity(bc->irq, bc->cpumask);
}

static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
                                     ktime_t expires)
{
        if (!clockevent_state_oneshot(bc))
                clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);

        clockevents_program_event(bc, expires, 1);
        tick_broadcast_set_affinity(bc, cpumask_of(cpu));
}

static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
{
        clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
}

/*
 * Called from irq_enter() when idle was interrupted to reenable the
 * per cpu device.
 */
void tick_check_oneshot_broadcast_this_cpu(void)
{
        if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
                struct tick_device *td = this_cpu_ptr(&tick_cpu_device);

                /*
                 * We might be in the middle of switching over from
                 * periodic to oneshot. If the CPU has not yet
                 * switched over, leave the device alone.
                 */
                if (td->mode == TICKDEV_MODE_ONESHOT) {
                        clockevents_switch_state(td->evtdev,
                                              CLOCK_EVT_STATE_ONESHOT);
                }
        }
}

/*
 * Handle oneshot mode broadcasting
 */
static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
{
        struct tick_device *td;
        ktime_t now, next_event;
        int cpu, next_cpu = 0;
        bool bc_local;

        raw_spin_lock(&tick_broadcast_lock);
        dev->next_event = KTIME_MAX;
        next_event = KTIME_MAX;
        cpumask_clear(tmpmask);
        now = ktime_get();
        /* Find all expired events */
        for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
                /*
                 * Required for !SMP because for_each_cpu() reports
                 * unconditionally CPU0 as set on UP kernels.
                 */
                if (!IS_ENABLED(CONFIG_SMP) &&
                    cpumask_empty(tick_broadcast_oneshot_mask))
                        break;

                td = &per_cpu(tick_cpu_device, cpu);
                if (td->evtdev->next_event <= now) {
                        cpumask_set_cpu(cpu, tmpmask);
                        /*
                         * Mark the remote cpu in the pending mask, so
                         * it can avoid reprogramming the cpu local
                         * timer in tick_broadcast_oneshot_control().
                         */
                        cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
                } else if (td->evtdev->next_event < next_event) {
                        next_event = td->evtdev->next_event;
                        next_cpu = cpu;
                }
        }

        /*
         * Remove the current cpu from the pending mask. The event is
         * delivered immediately in tick_do_broadcast() !
         */
        cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);

        /* Take care of enforced broadcast requests */
        cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
        cpumask_clear(tick_broadcast_force_mask);

        /*
         * Sanity check. Catch the case where we try to broadcast to
         * offline cpus.
         */
        if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
                cpumask_and(tmpmask, tmpmask, cpu_online_mask);

        /*
         * Wakeup the cpus which have an expired event.
         */
        bc_local = tick_do_broadcast(tmpmask);

        /*
         * Two reasons for reprogram:
         *
         * - The global event did not expire any CPU local
         * events. This happens in dyntick mode, as the maximum PIT
         * delta is quite small.
         *
         * - There are pending events on sleeping CPUs which were not
         * in the event mask
         */
        if (next_event != KTIME_MAX)
                tick_broadcast_set_event(dev, next_cpu, next_event);

        raw_spin_unlock(&tick_broadcast_lock);

        if (bc_local) {
                td = this_cpu_ptr(&tick_cpu_device);
                td->evtdev->event_handler(td->evtdev);
        }
}

static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
{
        if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
                return 0;
        if (bc->next_event == KTIME_MAX)
                return 0;
        return bc->bound_on == cpu ? -EBUSY : 0;
}

static void broadcast_shutdown_local(struct clock_event_device *bc,
                                     struct clock_event_device *dev)
{
        /*
         * For hrtimer based broadcasting we cannot shutdown the cpu
         * local device if our own event is the first one to expire or
         * if we own the broadcast timer.
         */
        if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
                if (broadcast_needs_cpu(bc, smp_processor_id()))
                        return;
                if (dev->next_event < bc->next_event)
                        return;
        }
        clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
}

static int ___tick_broadcast_oneshot_control(enum tick_broadcast_state state,
                                             struct tick_device *td,
                                             int cpu)
{
        struct clock_event_device *bc, *dev = td->evtdev;
        int ret = 0;
        ktime_t now;

        raw_spin_lock(&tick_broadcast_lock);
        bc = tick_broadcast_device.evtdev;

        if (state == TICK_BROADCAST_ENTER) {
                /*
                 * If the current CPU owns the hrtimer broadcast
                 * mechanism, it cannot go deep idle and we do not add
                 * the CPU to the broadcast mask. We don't have to go
                 * through the EXIT path as the local timer is not
                 * shutdown.
                 */
                ret = broadcast_needs_cpu(bc, cpu);
                if (ret)
                        goto out;

                /*
                 * If the broadcast device is in periodic mode, we
                 * return.
                 */
                if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
                        /* If it is a hrtimer based broadcast, return busy */
                        if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
                                ret = -EBUSY;
                        goto out;
                }

                if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
                        WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));

                        /* Conditionally shut down the local timer. */
                        broadcast_shutdown_local(bc, dev);

                        /*
                         * We only reprogram the broadcast timer if we
                         * did not mark ourself in the force mask and
                         * if the cpu local event is earlier than the
                         * broadcast event. If the current CPU is in
                         * the force mask, then we are going to be
                         * woken by the IPI right away; we return
                         * busy, so the CPU does not try to go deep
                         * idle.
                         */
                        if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
                                ret = -EBUSY;
                        } else if (dev->next_event < bc->next_event) {
                                tick_broadcast_set_event(bc, cpu, dev->next_event);
                                /*
                                 * In case of hrtimer broadcasts the
                                 * programming might have moved the
                                 * timer to this cpu. If yes, remove
                                 * us from the broadcast mask and
                                 * return busy.
                                 */
                                ret = broadcast_needs_cpu(bc, cpu);
                                if (ret) {
                                        cpumask_clear_cpu(cpu,
                                                tick_broadcast_oneshot_mask);
                                }
                        }
                }
        } else {
                if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
                        clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
                        /*
                         * The cpu which was handling the broadcast
                         * timer marked this cpu in the broadcast
                         * pending mask and fired the broadcast
                         * IPI. So we are going to handle the expired
                         * event anyway via the broadcast IPI
                         * handler. No need to reprogram the timer
                         * with an already expired event.
                         */
                        if (cpumask_test_and_clear_cpu(cpu,
                                       tick_broadcast_pending_mask))
                                goto out;

                        /*
                         * Bail out if there is no next event.
                         */
                        if (dev->next_event == KTIME_MAX)
                                goto out;
                        /*
                         * If the pending bit is not set, then we are
                         * either the CPU handling the broadcast
                         * interrupt or we got woken by something else.
                         *
                         * We are no longer in the broadcast mask, so
                         * if the cpu local expiry time is already
                         * reached, we would reprogram the cpu local
                         * timer with an already expired event.
                         *
                         * This can lead to a ping-pong when we return
                         * to idle and therefore rearm the broadcast
                         * timer before the cpu local timer was able
                         * to fire. This happens because the forced
                         * reprogramming makes sure that the event
                         * will happen in the future and depending on
                         * the min_delta setting this might be far
                         * enough out that the ping-pong starts.
                         *
                         * If the cpu local next_event has expired
                         * then we know that the broadcast timer
                         * next_event has expired as well and
                         * broadcast is about to be handled. So we
                         * avoid reprogramming and enforce that the
                         * broadcast handler, which did not run yet,
                         * will invoke the cpu local handler.
                         *
                         * We cannot call the handler directly from
                         * here, because we might be in a NOHZ phase
                         * and we did not go through the irq_enter()
                         * nohz fixups.
                         */
                        now = ktime_get();
                        if (dev->next_event <= now) {
                                cpumask_set_cpu(cpu, tick_broadcast_force_mask);
                                goto out;
                        }
                        /*
                         * We got woken by something else. Reprogram
                         * the cpu local timer device.
                         */
                        tick_program_event(dev->next_event, 1);
                }
        }
out:
        raw_spin_unlock(&tick_broadcast_lock);
        return ret;
}

static int tick_oneshot_wakeup_control(enum tick_broadcast_state state,
                                       struct tick_device *td,
                                       int cpu)
{
        struct clock_event_device *dev, *wd;

        dev = td->evtdev;
        if (td->mode != TICKDEV_MODE_ONESHOT)
                return -EINVAL;

        wd = tick_get_oneshot_wakeup_device(cpu);
        if (!wd)
                return -ENODEV;

        switch (state) {
        case TICK_BROADCAST_ENTER:
                clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT_STOPPED);
                clockevents_switch_state(wd, CLOCK_EVT_STATE_ONESHOT);
                clockevents_program_event(wd, dev->next_event, 1);
                break;
        case TICK_BROADCAST_EXIT:
                /* We may have transitioned to oneshot mode while idle */
                if (clockevent_get_state(wd) != CLOCK_EVT_STATE_ONESHOT)
                        return -ENODEV;
        }

        return 0;
}

int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
{
        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
        int cpu = smp_processor_id();

        if (!tick_oneshot_wakeup_control(state, td, cpu))
                return 0;

        if (tick_broadcast_device.evtdev)
                return ___tick_broadcast_oneshot_control(state, td, cpu);

        /*
         * If there is no broadcast or wakeup device, tell the caller not
         * to go into deep idle.
         */
        return -EBUSY;
}

/*
 * Reset the one shot broadcast for a cpu
 *
 * Called with tick_broadcast_lock held
 */
static void tick_broadcast_clear_oneshot(int cpu)
{
        cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
        cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
}

static void tick_broadcast_init_next_event(struct cpumask *mask,
                                           ktime_t expires)
{
        struct tick_device *td;
        int cpu;

        for_each_cpu(cpu, mask) {
                td = &per_cpu(tick_cpu_device, cpu);
                if (td->evtdev)
                        td->evtdev->next_event = expires;
        }
}

static inline ktime_t tick_get_next_period(void)
{
        ktime_t next;

        /*
         * Protect against concurrent updates (store /load tearing on
         * 32bit). It does not matter if the time is already in the
         * past. The broadcast device which is about to be programmed will
         * fire in any case.
         */
        raw_spin_lock(&jiffies_lock);
        next = tick_next_period;
        raw_spin_unlock(&jiffies_lock);
        return next;
}

/**
 * tick_broadcast_setup_oneshot - setup the broadcast device
 * @bc: the broadcast device
 * @from_periodic: true if called from periodic mode
 */
static void tick_broadcast_setup_oneshot(struct clock_event_device *bc,
                                         bool from_periodic)
{
        int cpu = smp_processor_id();
        ktime_t nexttick = 0;

        if (!bc)
                return;

        /*
         * When the broadcast device was switched to oneshot by the first
         * CPU handling the NOHZ change, the other CPUs will reach this
         * code via hrtimer_run_queues() -> tick_check_oneshot_change()
         * too. Set up the broadcast device only once!
         */
        if (bc->event_handler == tick_handle_oneshot_broadcast) {
                /*
                 * The CPU which switched from periodic to oneshot mode
                 * set the broadcast oneshot bit for all other CPUs which
                 * are in the general (periodic) broadcast mask to ensure
                 * that CPUs which wait for the periodic broadcast are
                 * woken up.
                 *
                 * Clear the bit for the local CPU as the set bit would
                 * prevent the first tick_broadcast_enter() after this CPU
                 * switched to oneshot state to program the broadcast
                 * device.
                 *
                 * This code can also be reached via tick_broadcast_control(),
                 * but this cannot avoid the tick_broadcast_clear_oneshot()
                 * as that would break the periodic to oneshot transition of
                 * secondary CPUs. But that's harmless as the below only
                 * clears already cleared bits.
                 */
                tick_broadcast_clear_oneshot(cpu);
                return;
        }


        bc->event_handler = tick_handle_oneshot_broadcast;
        bc->next_event = KTIME_MAX;

        /*
         * When the tick mode is switched from periodic to oneshot it must
         * be ensured that CPUs which are waiting for periodic broadcast
         * get their wake-up at the next tick.  This is achieved by ORing
         * tick_broadcast_mask into tick_broadcast_oneshot_mask.
         *
         * For other callers, e.g. broadcast device replacement,
         * tick_broadcast_oneshot_mask must not be touched as this would
         * set bits for CPUs which are already NOHZ, but not idle. Their
         * next tick_broadcast_enter() would observe the bit set and fail
         * to update the expiry time and the broadcast event device.
         */
        if (from_periodic) {
                cpumask_copy(tmpmask, tick_broadcast_mask);
                /* Remove the local CPU as it is obviously not idle */
                cpumask_clear_cpu(cpu, tmpmask);
                cpumask_or(tick_broadcast_oneshot_mask, tick_broadcast_oneshot_mask, tmpmask);

                /*
                 * Ensure that the oneshot broadcast handler will wake the
                 * CPUs which are still waiting for periodic broadcast.
                 */
                nexttick = tick_get_next_period();
                tick_broadcast_init_next_event(tmpmask, nexttick);

                /*
                 * If the underlying broadcast clock event device is
                 * already in oneshot state, then there is nothing to do.
                 * The device was already armed for the next tick
                 * in tick_handle_broadcast_periodic()
                 */
                if (clockevent_state_oneshot(bc))
                        return;
        }

        /*
         * When switching from periodic to oneshot mode arm the broadcast
         * device for the next tick.
         *
         * If the broadcast device has been replaced in oneshot mode and
         * the oneshot broadcast mask is not empty, then arm it to expire
         * immediately in order to reevaluate the next expiring timer.
         * @nexttick is 0 and therefore in the past which will cause the
         * clockevent code to force an event.
         *
         * For both cases the programming can be avoided when the oneshot
         * broadcast mask is empty.
         *
         * tick_broadcast_set_event() implicitly switches the broadcast
         * device to oneshot state.
         */
        if (!cpumask_empty(tick_broadcast_oneshot_mask))
                tick_broadcast_set_event(bc, cpu, nexttick);
}

/*
 * Select oneshot operating mode for the broadcast device
 */
void tick_broadcast_switch_to_oneshot(void)
{
        struct clock_event_device *bc;
        enum tick_device_mode oldmode;
        unsigned long flags;

        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);

        oldmode = tick_broadcast_device.mode;
        tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
        bc = tick_broadcast_device.evtdev;
        if (bc)
                tick_broadcast_setup_oneshot(bc, oldmode == TICKDEV_MODE_PERIODIC);

        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

#ifdef CONFIG_HOTPLUG_CPU
void hotplug_cpu__broadcast_tick_pull(int deadcpu)
{
        struct clock_event_device *bc;
        unsigned long flags;

        raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
        bc = tick_broadcast_device.evtdev;

        if (bc && broadcast_needs_cpu(bc, deadcpu)) {
                /*
                 * If the broadcast force bit of the current CPU is set,
                 * then the current CPU has not yet reprogrammed the local
                 * timer device to avoid a ping-pong race. See
                 * ___tick_broadcast_oneshot_control().
                 *
                 * If the broadcast device is hrtimer based then
                 * programming the broadcast event below does not have any
                 * effect because the local clockevent device is not
                 * running and not programmed because the broadcast event
                 * is not earlier than the pending event of the local clock
                 * event device. As a consequence all CPUs waiting for a
                 * broadcast event are stuck forever.
                 *
                 * Detect this condition and reprogram the cpu local timer
                 * device to avoid the starvation.
                 */
                if (tick_check_broadcast_expired()) {
                        struct tick_device *td = this_cpu_ptr(&tick_cpu_device);

                        cpumask_clear_cpu(smp_processor_id(), tick_broadcast_force_mask);
                        tick_program_event(td->evtdev->next_event, 1);
                }

                /* This moves the broadcast assignment to this CPU: */
                clockevents_program_event(bc, bc->next_event, 1);
        }
        raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
}

/*
 * Remove a dying CPU from broadcasting
 */
static void tick_broadcast_oneshot_offline(unsigned int cpu)
{
        if (tick_get_oneshot_wakeup_device(cpu))
                tick_set_oneshot_wakeup_device(NULL, cpu);

        /*
         * Clear the broadcast masks for the dead cpu, but do not stop
         * the broadcast device!
         */
        cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
        cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
        cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
}
#endif

/*
 * Check, whether the broadcast device is in one shot mode
 */
int tick_broadcast_oneshot_active(void)
{
        return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
}

/*
 * Check whether the broadcast device supports oneshot.
 */
bool tick_broadcast_oneshot_available(void)
{
        struct clock_event_device *bc = tick_broadcast_device.evtdev;

        return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
}

#else
int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
{
        struct clock_event_device *bc = tick_broadcast_device.evtdev;

        if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
                return -EBUSY;

        return 0;
}
#endif

void __init tick_broadcast_init(void)
{
        zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
        zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
        zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
#ifdef CONFIG_TICK_ONESHOT
        zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
        zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
        zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
#endif
}

























































































































































































































































































































































    5 



    5 



    5 













    6 

















   51 






   40 

    2 












    8 





    3 





    2 













    2 


    5 















    4 






    4 






    5 











    2 


    2 








   18 

   19 











    3 






   23 


    8 






    1 


    2 






    5 











   13 






    1 


    1 










    2 






    6 










    2 






    1 








    5 

    4 











    2 



    6 





    3 





    8 





    5 


    5 





   22 







   22 








   10 






    6 






































    2 







    1 

    1 















































































   61 
    1 


    1 




   51 













    1 

   13 






















































































































    1 







    1 











































































































    1 






    1 





















































































































































































    1 


    1 








































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
 *
 * Development of this code funded by Astaro AG (http://www.astaro.com/)
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/netlink.h>
#include <linux/netfilter.h>
#include <linux/netfilter/nf_tables.h>
#include <net/netfilter/nf_tables_core.h>
#include <net/netfilter/nf_conntrack.h>
#include <net/netfilter/nf_conntrack_acct.h>
#include <net/netfilter/nf_conntrack_tuple.h>
#include <net/netfilter/nf_conntrack_helper.h>
#include <net/netfilter/nf_conntrack_ecache.h>
#include <net/netfilter/nf_conntrack_labels.h>
#include <net/netfilter/nf_conntrack_timeout.h>
#include <net/netfilter/nf_conntrack_l4proto.h>
#include <net/netfilter/nf_conntrack_expect.h>

struct nft_ct_helper_obj  {
        struct nf_conntrack_helper *helper4;
        struct nf_conntrack_helper *helper6;
        u8 l4proto;
};

#ifdef CONFIG_NF_CONNTRACK_ZONES
static DEFINE_PER_CPU(struct nf_conn *, nft_ct_pcpu_template);
static unsigned int nft_ct_pcpu_template_refcnt __read_mostly;
static DEFINE_MUTEX(nft_ct_pcpu_mutex);
#endif

static u64 nft_ct_get_eval_counter(const struct nf_conn_counter *c,
                                   enum nft_ct_keys k,
                                   enum ip_conntrack_dir d)
{
        if (d < IP_CT_DIR_MAX)
                return k == NFT_CT_BYTES ? atomic64_read(&c[d].bytes) :
                                           atomic64_read(&c[d].packets);

        return nft_ct_get_eval_counter(c, k, IP_CT_DIR_ORIGINAL) +
               nft_ct_get_eval_counter(c, k, IP_CT_DIR_REPLY);
}

static void nft_ct_get_eval(const struct nft_expr *expr,
                            struct nft_regs *regs,
                            const struct nft_pktinfo *pkt)
{
        const struct nft_ct *priv = nft_expr_priv(expr);
        u32 *dest = &regs->data[priv->dreg];
        enum ip_conntrack_info ctinfo;
        const struct nf_conn *ct;
        const struct nf_conn_help *help;
        const struct nf_conntrack_tuple *tuple;
        const struct nf_conntrack_helper *helper;
        unsigned int state;

        ct = nf_ct_get(pkt->skb, &ctinfo);

        switch (priv->key) {
        case NFT_CT_STATE:
                if (ct)
                        state = NF_CT_STATE_BIT(ctinfo);
                else if (ctinfo == IP_CT_UNTRACKED)
                        state = NF_CT_STATE_UNTRACKED_BIT;
                else
                        state = NF_CT_STATE_INVALID_BIT;
                *dest = state;
                return;
        default:
                break;
        }

        if (ct == NULL)
                goto err;

        switch (priv->key) {
        case NFT_CT_DIRECTION:
                nft_reg_store8(dest, CTINFO2DIR(ctinfo));
                return;
        case NFT_CT_STATUS:
                *dest = ct->status;
                return;
#ifdef CONFIG_NF_CONNTRACK_MARK
        case NFT_CT_MARK:
                *dest = READ_ONCE(ct->mark);
                return;
#endif
#ifdef CONFIG_NF_CONNTRACK_SECMARK
        case NFT_CT_SECMARK:
                *dest = ct->secmark;
                return;
#endif
        case NFT_CT_EXPIRATION:
                *dest = jiffies_to_msecs(nf_ct_expires(ct));
                return;
        case NFT_CT_HELPER:
                if (ct->master == NULL)
                        goto err;
                help = nfct_help(ct->master);
                if (help == NULL)
                        goto err;
                helper = rcu_dereference(help->helper);
                if (helper == NULL)
                        goto err;
                strscpy_pad((char *)dest, helper->name, NF_CT_HELPER_NAME_LEN);
                return;
#ifdef CONFIG_NF_CONNTRACK_LABELS
        case NFT_CT_LABELS: {
                struct nf_conn_labels *labels = nf_ct_labels_find(ct);

                if (labels)
                        memcpy(dest, labels->bits, NF_CT_LABELS_MAX_SIZE);
                else
                        memset(dest, 0, NF_CT_LABELS_MAX_SIZE);
                return;
        }
#endif
        case NFT_CT_BYTES:
        case NFT_CT_PKTS: {
                const struct nf_conn_acct *acct = nf_conn_acct_find(ct);
                u64 count = 0;

                if (acct)
                        count = nft_ct_get_eval_counter(acct->counter,
                                                        priv->key, priv->dir);
                memcpy(dest, &count, sizeof(count));
                return;
        }
        case NFT_CT_AVGPKT: {
                const struct nf_conn_acct *acct = nf_conn_acct_find(ct);
                u64 avgcnt = 0, bcnt = 0, pcnt = 0;

                if (acct) {
                        pcnt = nft_ct_get_eval_counter(acct->counter,
                                                       NFT_CT_PKTS, priv->dir);
                        bcnt = nft_ct_get_eval_counter(acct->counter,
                                                       NFT_CT_BYTES, priv->dir);
                        if (pcnt != 0)
                                avgcnt = div64_u64(bcnt, pcnt);
                }

                memcpy(dest, &avgcnt, sizeof(avgcnt));
                return;
        }
        case NFT_CT_L3PROTOCOL:
                nft_reg_store8(dest, nf_ct_l3num(ct));
                return;
        case NFT_CT_PROTOCOL:
                nft_reg_store8(dest, nf_ct_protonum(ct));
                return;
#ifdef CONFIG_NF_CONNTRACK_ZONES
        case NFT_CT_ZONE: {
                const struct nf_conntrack_zone *zone = nf_ct_zone(ct);
                u16 zoneid;

                if (priv->dir < IP_CT_DIR_MAX)
                        zoneid = nf_ct_zone_id(zone, priv->dir);
                else
                        zoneid = zone->id;

                nft_reg_store16(dest, zoneid);
                return;
        }
#endif
        case NFT_CT_ID:
                *dest = nf_ct_get_id(ct);
                return;
        default:
                break;
        }

        tuple = &ct->tuplehash[priv->dir].tuple;
        switch (priv->key) {
        case NFT_CT_SRC:
                memcpy(dest, tuple->src.u3.all,
                       nf_ct_l3num(ct) == NFPROTO_IPV4 ? 4 : 16);
                return;
        case NFT_CT_DST:
                memcpy(dest, tuple->dst.u3.all,
                       nf_ct_l3num(ct) == NFPROTO_IPV4 ? 4 : 16);
                return;
        case NFT_CT_PROTO_SRC:
                nft_reg_store16(dest, (__force u16)tuple->src.u.all);
                return;
        case NFT_CT_PROTO_DST:
                nft_reg_store16(dest, (__force u16)tuple->dst.u.all);
                return;
        case NFT_CT_SRC_IP:
                if (nf_ct_l3num(ct) != NFPROTO_IPV4)
                        goto err;
                *dest = (__force __u32)tuple->src.u3.ip;
                return;
        case NFT_CT_DST_IP:
                if (nf_ct_l3num(ct) != NFPROTO_IPV4)
                        goto err;
                *dest = (__force __u32)tuple->dst.u3.ip;
                return;
        case NFT_CT_SRC_IP6:
                if (nf_ct_l3num(ct) != NFPROTO_IPV6)
                        goto err;
                memcpy(dest, tuple->src.u3.ip6, sizeof(struct in6_addr));
                return;
        case NFT_CT_DST_IP6:
                if (nf_ct_l3num(ct) != NFPROTO_IPV6)
                        goto err;
                memcpy(dest, tuple->dst.u3.ip6, sizeof(struct in6_addr));
                return;
        default:
                break;
        }
        return;
err:
        regs->verdict.code = NFT_BREAK;
}

#ifdef CONFIG_NF_CONNTRACK_ZONES
static void nft_ct_set_zone_eval(const struct nft_expr *expr,
                                 struct nft_regs *regs,
                                 const struct nft_pktinfo *pkt)
{
        struct nf_conntrack_zone zone = { .dir = NF_CT_DEFAULT_ZONE_DIR };
        const struct nft_ct *priv = nft_expr_priv(expr);
        struct sk_buff *skb = pkt->skb;
        enum ip_conntrack_info ctinfo;
        u16 value = nft_reg_load16(&regs->data[priv->sreg]);
        struct nf_conn *ct;
        int oldcnt;

        ct = nf_ct_get(skb, &ctinfo);
        if (ct) /* already tracked */
                return;

        zone.id = value;

        switch (priv->dir) {
        case IP_CT_DIR_ORIGINAL:
                zone.dir = NF_CT_ZONE_DIR_ORIG;
                break;
        case IP_CT_DIR_REPLY:
                zone.dir = NF_CT_ZONE_DIR_REPL;
                break;
        default:
                break;
        }

        ct = this_cpu_read(nft_ct_pcpu_template);

        __refcount_inc(&ct->ct_general.use, &oldcnt);
        if (likely(oldcnt == 1)) {
                nf_ct_zone_add(ct, &zone);
        } else {
                refcount_dec(&ct->ct_general.use);
                /* previous skb got queued to userspace, allocate temporary
                 * one until percpu template can be reused.
                 */
                ct = nf_ct_tmpl_alloc(nft_net(pkt), &zone, GFP_ATOMIC);
                if (!ct) {
                        regs->verdict.code = NF_DROP;
                        return;
                }
                __set_bit(IPS_CONFIRMED_BIT, &ct->status);
        }

        nf_ct_set(skb, ct, IP_CT_NEW);
}
#endif

static void nft_ct_set_eval(const struct nft_expr *expr,
                            struct nft_regs *regs,
                            const struct nft_pktinfo *pkt)
{
        const struct nft_ct *priv = nft_expr_priv(expr);
        struct sk_buff *skb = pkt->skb;
#if defined(CONFIG_NF_CONNTRACK_MARK) || defined(CONFIG_NF_CONNTRACK_SECMARK)
        u32 value = regs->data[priv->sreg];
#endif
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;

        ct = nf_ct_get(skb, &ctinfo);
        if (ct == NULL || nf_ct_is_template(ct))
                return;

        switch (priv->key) {
#ifdef CONFIG_NF_CONNTRACK_MARK
        case NFT_CT_MARK:
                if (READ_ONCE(ct->mark) != value) {
                        WRITE_ONCE(ct->mark, value);
                        nf_conntrack_event_cache(IPCT_MARK, ct);
                }
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_SECMARK
        case NFT_CT_SECMARK:
                if (ct->secmark != value) {
                        ct->secmark = value;
                        nf_conntrack_event_cache(IPCT_SECMARK, ct);
                }
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_LABELS
        case NFT_CT_LABELS:
                nf_connlabels_replace(ct,
                                      &regs->data[priv->sreg],
                                      &regs->data[priv->sreg],
                                      NF_CT_LABELS_MAX_SIZE / sizeof(u32));
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_EVENTS
        case NFT_CT_EVENTMASK: {
                struct nf_conntrack_ecache *e = nf_ct_ecache_find(ct);
                u32 ctmask = regs->data[priv->sreg];

                if (e) {
                        if (e->ctmask != ctmask)
                                e->ctmask = ctmask;
                        break;
                }

                if (ctmask && !nf_ct_is_confirmed(ct))
                        nf_ct_ecache_ext_add(ct, ctmask, 0, GFP_ATOMIC);
                break;
        }
#endif
        default:
                break;
        }
}

static const struct nla_policy nft_ct_policy[NFTA_CT_MAX + 1] = {
        [NFTA_CT_DREG]                = { .type = NLA_U32 },
        [NFTA_CT_KEY]                = NLA_POLICY_MAX(NLA_BE32, 255),
        [NFTA_CT_DIRECTION]        = { .type = NLA_U8 },
        [NFTA_CT_SREG]                = { .type = NLA_U32 },
};

#ifdef CONFIG_NF_CONNTRACK_ZONES
static void nft_ct_tmpl_put_pcpu(void)
{
        struct nf_conn *ct;
        int cpu;

        for_each_possible_cpu(cpu) {
                ct = per_cpu(nft_ct_pcpu_template, cpu);
                if (!ct)
                        break;
                nf_ct_put(ct);
                per_cpu(nft_ct_pcpu_template, cpu) = NULL;
        }
}

static bool nft_ct_tmpl_alloc_pcpu(void)
{
        struct nf_conntrack_zone zone = { .id = 0 };
        struct nf_conn *tmp;
        int cpu;

        if (nft_ct_pcpu_template_refcnt)
                return true;

        for_each_possible_cpu(cpu) {
                tmp = nf_ct_tmpl_alloc(&init_net, &zone, GFP_KERNEL);
                if (!tmp) {
                        nft_ct_tmpl_put_pcpu();
                        return false;
                }

                __set_bit(IPS_CONFIRMED_BIT, &tmp->status);
                per_cpu(nft_ct_pcpu_template, cpu) = tmp;
        }

        return true;
}
#endif

static int nft_ct_get_init(const struct nft_ctx *ctx,
                           const struct nft_expr *expr,
                           const struct nlattr * const tb[])
{
        struct nft_ct *priv = nft_expr_priv(expr);
        unsigned int len;
        int err;

        priv->key = ntohl(nla_get_be32(tb[NFTA_CT_KEY]));
        priv->dir = IP_CT_DIR_MAX;
        switch (priv->key) {
        case NFT_CT_DIRECTION:
                if (tb[NFTA_CT_DIRECTION] != NULL)
                        return -EINVAL;
                len = sizeof(u8);
                break;
        case NFT_CT_STATE:
        case NFT_CT_STATUS:
#ifdef CONFIG_NF_CONNTRACK_MARK
        case NFT_CT_MARK:
#endif
#ifdef CONFIG_NF_CONNTRACK_SECMARK
        case NFT_CT_SECMARK:
#endif
        case NFT_CT_EXPIRATION:
                if (tb[NFTA_CT_DIRECTION] != NULL)
                        return -EINVAL;
                len = sizeof(u32);
                break;
#ifdef CONFIG_NF_CONNTRACK_LABELS
        case NFT_CT_LABELS:
                if (tb[NFTA_CT_DIRECTION] != NULL)
                        return -EINVAL;
                len = NF_CT_LABELS_MAX_SIZE;
                break;
#endif
        case NFT_CT_HELPER:
                if (tb[NFTA_CT_DIRECTION] != NULL)
                        return -EINVAL;
                len = NF_CT_HELPER_NAME_LEN;
                break;

        case NFT_CT_L3PROTOCOL:
        case NFT_CT_PROTOCOL:
                /* For compatibility, do not report error if NFTA_CT_DIRECTION
                 * attribute is specified.
                 */
                len = sizeof(u8);
                break;
        case NFT_CT_SRC:
        case NFT_CT_DST:
                if (tb[NFTA_CT_DIRECTION] == NULL)
                        return -EINVAL;

                switch (ctx->family) {
                case NFPROTO_IPV4:
                        len = sizeof_field(struct nf_conntrack_tuple,
                                           src.u3.ip);
                        break;
                case NFPROTO_IPV6:
                case NFPROTO_INET:
                        len = sizeof_field(struct nf_conntrack_tuple,
                                           src.u3.ip6);
                        break;
                default:
                        return -EAFNOSUPPORT;
                }
                break;
        case NFT_CT_SRC_IP:
        case NFT_CT_DST_IP:
                if (tb[NFTA_CT_DIRECTION] == NULL)
                        return -EINVAL;

                len = sizeof_field(struct nf_conntrack_tuple, src.u3.ip);
                break;
        case NFT_CT_SRC_IP6:
        case NFT_CT_DST_IP6:
                if (tb[NFTA_CT_DIRECTION] == NULL)
                        return -EINVAL;

                len = sizeof_field(struct nf_conntrack_tuple, src.u3.ip6);
                break;
        case NFT_CT_PROTO_SRC:
        case NFT_CT_PROTO_DST:
                if (tb[NFTA_CT_DIRECTION] == NULL)
                        return -EINVAL;
                len = sizeof_field(struct nf_conntrack_tuple, src.u.all);
                break;
        case NFT_CT_BYTES:
        case NFT_CT_PKTS:
        case NFT_CT_AVGPKT:
                len = sizeof(u64);
                break;
#ifdef CONFIG_NF_CONNTRACK_ZONES
        case NFT_CT_ZONE:
                len = sizeof(u16);
                break;
#endif
        case NFT_CT_ID:
                if (tb[NFTA_CT_DIRECTION])
                        return -EINVAL;

                len = sizeof(u32);
                break;
        default:
                return -EOPNOTSUPP;
        }

        if (tb[NFTA_CT_DIRECTION] != NULL) {
                priv->dir = nla_get_u8(tb[NFTA_CT_DIRECTION]);
                switch (priv->dir) {
                case IP_CT_DIR_ORIGINAL:
                case IP_CT_DIR_REPLY:
                        break;
                default:
                        return -EINVAL;
                }
        }

        priv->len = len;
        err = nft_parse_register_store(ctx, tb[NFTA_CT_DREG], &priv->dreg, NULL,
                                       NFT_DATA_VALUE, len);
        if (err < 0)
                return err;

        err = nf_ct_netns_get(ctx->net, ctx->family);
        if (err < 0)
                return err;

        if (priv->key == NFT_CT_BYTES ||
            priv->key == NFT_CT_PKTS  ||
            priv->key == NFT_CT_AVGPKT)
                nf_ct_set_acct(ctx->net, true);

        return 0;
}

static void __nft_ct_set_destroy(const struct nft_ctx *ctx, struct nft_ct *priv)
{
        switch (priv->key) {
#ifdef CONFIG_NF_CONNTRACK_LABELS
        case NFT_CT_LABELS:
                nf_connlabels_put(ctx->net);
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_ZONES
        case NFT_CT_ZONE:
                mutex_lock(&nft_ct_pcpu_mutex);
                if (--nft_ct_pcpu_template_refcnt == 0)
                        nft_ct_tmpl_put_pcpu();
                mutex_unlock(&nft_ct_pcpu_mutex);
                break;
#endif
        default:
                break;
        }
}

static int nft_ct_set_init(const struct nft_ctx *ctx,
                           const struct nft_expr *expr,
                           const struct nlattr * const tb[])
{
        struct nft_ct *priv = nft_expr_priv(expr);
        unsigned int len;
        int err;

        priv->dir = IP_CT_DIR_MAX;
        priv->key = ntohl(nla_get_be32(tb[NFTA_CT_KEY]));
        switch (priv->key) {
#ifdef CONFIG_NF_CONNTRACK_MARK
        case NFT_CT_MARK:
                if (tb[NFTA_CT_DIRECTION])
                        return -EINVAL;
                len = sizeof_field(struct nf_conn, mark);
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_LABELS
        case NFT_CT_LABELS:
                if (tb[NFTA_CT_DIRECTION])
                        return -EINVAL;
                len = NF_CT_LABELS_MAX_SIZE;
                err = nf_connlabels_get(ctx->net, (len * BITS_PER_BYTE) - 1);
                if (err)
                        return err;
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_ZONES
        case NFT_CT_ZONE:
                mutex_lock(&nft_ct_pcpu_mutex);
                if (!nft_ct_tmpl_alloc_pcpu()) {
                        mutex_unlock(&nft_ct_pcpu_mutex);
                        return -ENOMEM;
                }
                nft_ct_pcpu_template_refcnt++;
                mutex_unlock(&nft_ct_pcpu_mutex);
                len = sizeof(u16);
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_EVENTS
        case NFT_CT_EVENTMASK:
                if (tb[NFTA_CT_DIRECTION])
                        return -EINVAL;
                len = sizeof(u32);
                break;
#endif
#ifdef CONFIG_NF_CONNTRACK_SECMARK
        case NFT_CT_SECMARK:
                if (tb[NFTA_CT_DIRECTION])
                        return -EINVAL;
                len = sizeof(u32);
                break;
#endif
        default:
                return -EOPNOTSUPP;
        }

        if (tb[NFTA_CT_DIRECTION]) {
                priv->dir = nla_get_u8(tb[NFTA_CT_DIRECTION]);
                switch (priv->dir) {
                case IP_CT_DIR_ORIGINAL:
                case IP_CT_DIR_REPLY:
                        break;
                default:
                        err = -EINVAL;
                        goto err1;
                }
        }

        priv->len = len;
        err = nft_parse_register_load(ctx, tb[NFTA_CT_SREG], &priv->sreg, len);
        if (err < 0)
                goto err1;

        err = nf_ct_netns_get(ctx->net, ctx->family);
        if (err < 0)
                goto err1;

        return 0;

err1:
        __nft_ct_set_destroy(ctx, priv);
        return err;
}

static void nft_ct_get_destroy(const struct nft_ctx *ctx,
                               const struct nft_expr *expr)
{
        nf_ct_netns_put(ctx->net, ctx->family);
}

static void nft_ct_set_destroy(const struct nft_ctx *ctx,
                               const struct nft_expr *expr)
{
        struct nft_ct *priv = nft_expr_priv(expr);

        __nft_ct_set_destroy(ctx, priv);
        nf_ct_netns_put(ctx->net, ctx->family);
}

static int nft_ct_get_dump(struct sk_buff *skb,
                           const struct nft_expr *expr, bool reset)
{
        const struct nft_ct *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_CT_DREG, priv->dreg))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_CT_KEY, htonl(priv->key)))
                goto nla_put_failure;

        switch (priv->key) {
        case NFT_CT_SRC:
        case NFT_CT_DST:
        case NFT_CT_SRC_IP:
        case NFT_CT_DST_IP:
        case NFT_CT_SRC_IP6:
        case NFT_CT_DST_IP6:
        case NFT_CT_PROTO_SRC:
        case NFT_CT_PROTO_DST:
                if (nla_put_u8(skb, NFTA_CT_DIRECTION, priv->dir))
                        goto nla_put_failure;
                break;
        case NFT_CT_BYTES:
        case NFT_CT_PKTS:
        case NFT_CT_AVGPKT:
        case NFT_CT_ZONE:
                if (priv->dir < IP_CT_DIR_MAX &&
                    nla_put_u8(skb, NFTA_CT_DIRECTION, priv->dir))
                        goto nla_put_failure;
                break;
        default:
                break;
        }

        return 0;

nla_put_failure:
        return -1;
}

static bool nft_ct_get_reduce(struct nft_regs_track *track,
                              const struct nft_expr *expr)
{
        const struct nft_ct *priv = nft_expr_priv(expr);
        const struct nft_ct *ct;

        if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        ct = nft_expr_priv(track->regs[priv->dreg].selector);
        if (priv->key != ct->key) {
                nft_reg_track_update(track, expr, priv->dreg, priv->len);
                return false;
        }

        if (!track->regs[priv->dreg].bitwise)
                return true;

        return nft_expr_reduce_bitwise(track, expr);
}

static int nft_ct_set_dump(struct sk_buff *skb,
                           const struct nft_expr *expr, bool reset)
{
        const struct nft_ct *priv = nft_expr_priv(expr);

        if (nft_dump_register(skb, NFTA_CT_SREG, priv->sreg))
                goto nla_put_failure;
        if (nla_put_be32(skb, NFTA_CT_KEY, htonl(priv->key)))
                goto nla_put_failure;

        switch (priv->key) {
        case NFT_CT_ZONE:
                if (priv->dir < IP_CT_DIR_MAX &&
                    nla_put_u8(skb, NFTA_CT_DIRECTION, priv->dir))
                        goto nla_put_failure;
                break;
        default:
                break;
        }

        return 0;

nla_put_failure:
        return -1;
}

static struct nft_expr_type nft_ct_type;
static const struct nft_expr_ops nft_ct_get_ops = {
        .type                = &nft_ct_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
        .eval                = nft_ct_get_eval,
        .init                = nft_ct_get_init,
        .destroy        = nft_ct_get_destroy,
        .dump                = nft_ct_get_dump,
        .reduce                = nft_ct_get_reduce,
};

static bool nft_ct_set_reduce(struct nft_regs_track *track,
                              const struct nft_expr *expr)
{
        int i;

        for (i = 0; i < NFT_REG32_NUM; i++) {
                if (!track->regs[i].selector)
                        continue;

                if (track->regs[i].selector->ops != &nft_ct_get_ops)
                        continue;

                __nft_reg_track_cancel(track, i);
        }

        return false;
}

#ifdef CONFIG_MITIGATION_RETPOLINE
static const struct nft_expr_ops nft_ct_get_fast_ops = {
        .type                = &nft_ct_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
        .eval                = nft_ct_get_fast_eval,
        .init                = nft_ct_get_init,
        .destroy        = nft_ct_get_destroy,
        .dump                = nft_ct_get_dump,
        .reduce                = nft_ct_set_reduce,
};
#endif

static const struct nft_expr_ops nft_ct_set_ops = {
        .type                = &nft_ct_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
        .eval                = nft_ct_set_eval,
        .init                = nft_ct_set_init,
        .destroy        = nft_ct_set_destroy,
        .dump                = nft_ct_set_dump,
        .reduce                = nft_ct_set_reduce,
};

#ifdef CONFIG_NF_CONNTRACK_ZONES
static const struct nft_expr_ops nft_ct_set_zone_ops = {
        .type                = &nft_ct_type,
        .size                = NFT_EXPR_SIZE(sizeof(struct nft_ct)),
        .eval                = nft_ct_set_zone_eval,
        .init                = nft_ct_set_init,
        .destroy        = nft_ct_set_destroy,
        .dump                = nft_ct_set_dump,
        .reduce                = nft_ct_set_reduce,
};
#endif

static const struct nft_expr_ops *
nft_ct_select_ops(const struct nft_ctx *ctx,
                    const struct nlattr * const tb[])
{
        if (tb[NFTA_CT_KEY] == NULL)
                return ERR_PTR(-EINVAL);

        if (tb[NFTA_CT_DREG] && tb[NFTA_CT_SREG])
                return ERR_PTR(-EINVAL);

        if (tb[NFTA_CT_DREG]) {
#ifdef CONFIG_MITIGATION_RETPOLINE
                u32 k = ntohl(nla_get_be32(tb[NFTA_CT_KEY]));

                switch (k) {
                case NFT_CT_STATE:
                case NFT_CT_DIRECTION:
                case NFT_CT_STATUS:
                case NFT_CT_MARK:
                case NFT_CT_SECMARK:
                        return &nft_ct_get_fast_ops;
                }
#endif
                return &nft_ct_get_ops;
        }

        if (tb[NFTA_CT_SREG]) {
#ifdef CONFIG_NF_CONNTRACK_ZONES
                if (nla_get_be32(tb[NFTA_CT_KEY]) == htonl(NFT_CT_ZONE))
                        return &nft_ct_set_zone_ops;
#endif
                return &nft_ct_set_ops;
        }

        return ERR_PTR(-EINVAL);
}

static struct nft_expr_type nft_ct_type __read_mostly = {
        .name                = "ct",
        .select_ops        = nft_ct_select_ops,
        .policy                = nft_ct_policy,
        .maxattr        = NFTA_CT_MAX,
        .owner                = THIS_MODULE,
};

static void nft_notrack_eval(const struct nft_expr *expr,
                             struct nft_regs *regs,
                             const struct nft_pktinfo *pkt)
{
        struct sk_buff *skb = pkt->skb;
        enum ip_conntrack_info ctinfo;
        struct nf_conn *ct;

        ct = nf_ct_get(pkt->skb, &ctinfo);
        /* Previously seen (loopback or untracked)?  Ignore. */
        if (ct || ctinfo == IP_CT_UNTRACKED)
                return;

        nf_ct_set(skb, ct, IP_CT_UNTRACKED);
}

static struct nft_expr_type nft_notrack_type;
static const struct nft_expr_ops nft_notrack_ops = {
        .type                = &nft_notrack_type,
        .size                = NFT_EXPR_SIZE(0),
        .eval                = nft_notrack_eval,
        .reduce                = NFT_REDUCE_READONLY,
};

static struct nft_expr_type nft_notrack_type __read_mostly = {
        .name                = "notrack",
        .ops                = &nft_notrack_ops,
        .owner                = THIS_MODULE,
};

#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
static int
nft_ct_timeout_parse_policy(void *timeouts,
                            const struct nf_conntrack_l4proto *l4proto,
                            struct net *net, const struct nlattr *attr)
{
        struct nlattr **tb;
        int ret = 0;

        tb = kcalloc(l4proto->ctnl_timeout.nlattr_max + 1, sizeof(*tb),
                     GFP_KERNEL);

        if (!tb)
                return -ENOMEM;

        ret = nla_parse_nested_deprecated(tb,
                                          l4proto->ctnl_timeout.nlattr_max,
                                          attr,
                                          l4proto->ctnl_timeout.nla_policy,
                                          NULL);
        if (ret < 0)
                goto err;

        ret = l4proto->ctnl_timeout.nlattr_to_obj(tb, net, timeouts);

err:
        kfree(tb);
        return ret;
}

struct nft_ct_timeout_obj {
        struct nf_ct_timeout    *timeout;
        u8                        l4proto;
};

static void nft_ct_timeout_obj_eval(struct nft_object *obj,
                                    struct nft_regs *regs,
                                    const struct nft_pktinfo *pkt)
{
        const struct nft_ct_timeout_obj *priv = nft_obj_data(obj);
        struct nf_conn *ct = (struct nf_conn *)skb_nfct(pkt->skb);
        struct nf_conn_timeout *timeout;
        const unsigned int *values;

        if (priv->l4proto != pkt->tprot)
                return;

        if (!ct || nf_ct_is_template(ct) || nf_ct_is_confirmed(ct))
                return;

        timeout = nf_ct_timeout_find(ct);
        if (!timeout) {
                timeout = nf_ct_timeout_ext_add(ct, priv->timeout, GFP_ATOMIC);
                if (!timeout) {
                        regs->verdict.code = NF_DROP;
                        return;
                }
        }

        rcu_assign_pointer(timeout->timeout, priv->timeout);

        /* adjust the timeout as per 'new' state. ct is unconfirmed,
         * so the current timestamp must not be added.
         */
        values = nf_ct_timeout_data(timeout);
        if (values)
                nf_ct_refresh(ct, values[0]);
}

static int nft_ct_timeout_obj_init(const struct nft_ctx *ctx,
                                   const struct nlattr * const tb[],
                                   struct nft_object *obj)
{
        struct nft_ct_timeout_obj *priv = nft_obj_data(obj);
        const struct nf_conntrack_l4proto *l4proto;
        struct nf_ct_timeout *timeout;
        int l3num = ctx->family;
        __u8 l4num;
        int ret;

        if (!tb[NFTA_CT_TIMEOUT_L4PROTO] ||
            !tb[NFTA_CT_TIMEOUT_DATA])
                return -EINVAL;

        if (tb[NFTA_CT_TIMEOUT_L3PROTO])
                l3num = ntohs(nla_get_be16(tb[NFTA_CT_TIMEOUT_L3PROTO]));

        l4num = nla_get_u8(tb[NFTA_CT_TIMEOUT_L4PROTO]);
        priv->l4proto = l4num;

        l4proto = nf_ct_l4proto_find(l4num);

        if (l4proto->l4proto != l4num) {
                ret = -EOPNOTSUPP;
                goto err_proto_put;
        }

        timeout = kzalloc(sizeof(struct nf_ct_timeout) +
                          l4proto->ctnl_timeout.obj_size, GFP_KERNEL);
        if (timeout == NULL) {
                ret = -ENOMEM;
                goto err_proto_put;
        }

        ret = nft_ct_timeout_parse_policy(&timeout->data, l4proto, ctx->net,
                                          tb[NFTA_CT_TIMEOUT_DATA]);
        if (ret < 0)
                goto err_free_timeout;

        timeout->l3num = l3num;
        timeout->l4proto = l4proto;

        ret = nf_ct_netns_get(ctx->net, ctx->family);
        if (ret < 0)
                goto err_free_timeout;

        priv->timeout = timeout;
        return 0;

err_free_timeout:
        kfree(timeout);
err_proto_put:
        return ret;
}

static void nft_ct_timeout_obj_destroy(const struct nft_ctx *ctx,
                                       struct nft_object *obj)
{
        struct nft_ct_timeout_obj *priv = nft_obj_data(obj);
        struct nf_ct_timeout *timeout = priv->timeout;

        nf_ct_untimeout(ctx->net, timeout);
        nf_ct_netns_put(ctx->net, ctx->family);
        kfree(priv->timeout);
}

static int nft_ct_timeout_obj_dump(struct sk_buff *skb,
                                   struct nft_object *obj, bool reset)
{
        const struct nft_ct_timeout_obj *priv = nft_obj_data(obj);
        const struct nf_ct_timeout *timeout = priv->timeout;
        struct nlattr *nest_params;
        int ret;

        if (nla_put_u8(skb, NFTA_CT_TIMEOUT_L4PROTO, timeout->l4proto->l4proto) ||
            nla_put_be16(skb, NFTA_CT_TIMEOUT_L3PROTO, htons(timeout->l3num)))
                return -1;

        nest_params = nla_nest_start(skb, NFTA_CT_TIMEOUT_DATA);
        if (!nest_params)
                return -1;

        ret = timeout->l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->data);
        if (ret < 0)
                return -1;
        nla_nest_end(skb, nest_params);
        return 0;
}

static const struct nla_policy nft_ct_timeout_policy[NFTA_CT_TIMEOUT_MAX + 1] = {
        [NFTA_CT_TIMEOUT_L3PROTO] = {.type = NLA_U16 },
        [NFTA_CT_TIMEOUT_L4PROTO] = {.type = NLA_U8 },
        [NFTA_CT_TIMEOUT_DATA]          = {.type = NLA_NESTED },
};

static struct nft_object_type nft_ct_timeout_obj_type;

static const struct nft_object_ops nft_ct_timeout_obj_ops = {
        .type                = &nft_ct_timeout_obj_type,
        .size                = sizeof(struct nft_ct_timeout_obj),
        .eval                = nft_ct_timeout_obj_eval,
        .init                = nft_ct_timeout_obj_init,
        .destroy        = nft_ct_timeout_obj_destroy,
        .dump                = nft_ct_timeout_obj_dump,
};

static struct nft_object_type nft_ct_timeout_obj_type __read_mostly = {
        .type                = NFT_OBJECT_CT_TIMEOUT,
        .ops                = &nft_ct_timeout_obj_ops,
        .maxattr        = NFTA_CT_TIMEOUT_MAX,
        .policy                = nft_ct_timeout_policy,
        .owner                = THIS_MODULE,
};
#endif /* CONFIG_NF_CONNTRACK_TIMEOUT */

static int nft_ct_helper_obj_init(const struct nft_ctx *ctx,
                                  const struct nlattr * const tb[],
                                  struct nft_object *obj)
{
        struct nft_ct_helper_obj *priv = nft_obj_data(obj);
        struct nf_conntrack_helper *help4, *help6;
        char name[NF_CT_HELPER_NAME_LEN];
        int family = ctx->family;
        int err;

        if (!tb[NFTA_CT_HELPER_NAME] || !tb[NFTA_CT_HELPER_L4PROTO])
                return -EINVAL;

        priv->l4proto = nla_get_u8(tb[NFTA_CT_HELPER_L4PROTO]);
        if (!priv->l4proto)
                return -ENOENT;

        nla_strscpy(name, tb[NFTA_CT_HELPER_NAME], sizeof(name));

        if (tb[NFTA_CT_HELPER_L3PROTO])
                family = ntohs(nla_get_be16(tb[NFTA_CT_HELPER_L3PROTO]));

        help4 = NULL;
        help6 = NULL;

        switch (family) {
        case NFPROTO_IPV4:
                if (ctx->family == NFPROTO_IPV6)
                        return -EINVAL;

                help4 = nf_conntrack_helper_try_module_get(name, family,
                                                           priv->l4proto);
                break;
        case NFPROTO_IPV6:
                if (ctx->family == NFPROTO_IPV4)
                        return -EINVAL;

                help6 = nf_conntrack_helper_try_module_get(name, family,
                                                           priv->l4proto);
                break;
        case NFPROTO_NETDEV:
        case NFPROTO_BRIDGE:
        case NFPROTO_INET:
                help4 = nf_conntrack_helper_try_module_get(name, NFPROTO_IPV4,
                                                           priv->l4proto);
                help6 = nf_conntrack_helper_try_module_get(name, NFPROTO_IPV6,
                                                           priv->l4proto);
                break;
        default:
                return -EAFNOSUPPORT;
        }

        /* && is intentional; only error if INET found neither ipv4 or ipv6 */
        if (!help4 && !help6)
                return -ENOENT;

        priv->helper4 = help4;
        priv->helper6 = help6;

        err = nf_ct_netns_get(ctx->net, ctx->family);
        if (err < 0)
                goto err_put_helper;

        return 0;

err_put_helper:
        if (priv->helper4)
                nf_conntrack_helper_put(priv->helper4);
        if (priv->helper6)
                nf_conntrack_helper_put(priv->helper6);
        return err;
}

static void nft_ct_helper_obj_destroy(const struct nft_ctx *ctx,
                                      struct nft_object *obj)
{
        struct nft_ct_helper_obj *priv = nft_obj_data(obj);

        if (priv->helper4)
                nf_conntrack_helper_put(priv->helper4);
        if (priv->helper6)
                nf_conntrack_helper_put(priv->helper6);

        nf_ct_netns_put(ctx->net, ctx->family);
}

static void nft_ct_helper_obj_eval(struct nft_object *obj,
                                   struct nft_regs *regs,
                                   const struct nft_pktinfo *pkt)
{
        const struct nft_ct_helper_obj *priv = nft_obj_data(obj);
        struct nf_conn *ct = (struct nf_conn *)skb_nfct(pkt->skb);
        struct nf_conntrack_helper *to_assign = NULL;
        struct nf_conn_help *help;

        if (!ct ||
            nf_ct_is_confirmed(ct) ||
            nf_ct_is_template(ct) ||
            priv->l4proto != nf_ct_protonum(ct))
                return;

        switch (nf_ct_l3num(ct)) {
        case NFPROTO_IPV4:
                to_assign = priv->helper4;
                break;
        case NFPROTO_IPV6:
                to_assign = priv->helper6;
                break;
        default:
                WARN_ON_ONCE(1);
                return;
        }

        if (!to_assign)
                return;

        if (test_bit(IPS_HELPER_BIT, &ct->status))
                return;

        help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
        if (help) {
                rcu_assign_pointer(help->helper, to_assign);
                set_bit(IPS_HELPER_BIT, &ct->status);
        }
}

static int nft_ct_helper_obj_dump(struct sk_buff *skb,
                                  struct nft_object *obj, bool reset)
{
        const struct nft_ct_helper_obj *priv = nft_obj_data(obj);
        const struct nf_conntrack_helper *helper;
        u16 family;

        if (priv->helper4 && priv->helper6) {
                family = NFPROTO_INET;
                helper = priv->helper4;
        } else if (priv->helper6) {
                family = NFPROTO_IPV6;
                helper = priv->helper6;
        } else {
                family = NFPROTO_IPV4;
                helper = priv->helper4;
        }

        if (nla_put_string(skb, NFTA_CT_HELPER_NAME, helper->name))
                return -1;

        if (nla_put_u8(skb, NFTA_CT_HELPER_L4PROTO, priv->l4proto))
                return -1;

        if (nla_put_be16(skb, NFTA_CT_HELPER_L3PROTO, htons(family)))
                return -1;

        return 0;
}

static const struct nla_policy nft_ct_helper_policy[NFTA_CT_HELPER_MAX + 1] = {
        [NFTA_CT_HELPER_NAME] = { .type = NLA_STRING,
                                  .len = NF_CT_HELPER_NAME_LEN - 1 },
        [NFTA_CT_HELPER_L3PROTO] = { .type = NLA_U16 },
        [NFTA_CT_HELPER_L4PROTO] = { .type = NLA_U8 },
};

static struct nft_object_type nft_ct_helper_obj_type;
static const struct nft_object_ops nft_ct_helper_obj_ops = {
        .type                = &nft_ct_helper_obj_type,
        .size                = sizeof(struct nft_ct_helper_obj),
        .eval                = nft_ct_helper_obj_eval,
        .init                = nft_ct_helper_obj_init,
        .destroy        = nft_ct_helper_obj_destroy,
        .dump                = nft_ct_helper_obj_dump,
};

static struct nft_object_type nft_ct_helper_obj_type __read_mostly = {
        .type                = NFT_OBJECT_CT_HELPER,
        .ops                = &nft_ct_helper_obj_ops,
        .maxattr        = NFTA_CT_HELPER_MAX,
        .policy                = nft_ct_helper_policy,
        .owner                = THIS_MODULE,
};

struct nft_ct_expect_obj {
        u16                l3num;
        __be16                dport;
        u8                l4proto;
        u8                size;
        u32                timeout;
};

static int nft_ct_expect_obj_init(const struct nft_ctx *ctx,
                                  const struct nlattr * const tb[],
                                  struct nft_object *obj)
{
        struct nft_ct_expect_obj *priv = nft_obj_data(obj);

        if (!tb[NFTA_CT_EXPECT_L4PROTO] ||
            !tb[NFTA_CT_EXPECT_DPORT] ||
            !tb[NFTA_CT_EXPECT_TIMEOUT] ||
            !tb[NFTA_CT_EXPECT_SIZE])
                return -EINVAL;

        priv->l3num = ctx->family;
        if (tb[NFTA_CT_EXPECT_L3PROTO])
                priv->l3num = ntohs(nla_get_be16(tb[NFTA_CT_EXPECT_L3PROTO]));

        switch (priv->l3num) {
        case NFPROTO_IPV4:
        case NFPROTO_IPV6:
                if (priv->l3num == ctx->family || ctx->family == NFPROTO_INET)
                        break;

                return -EINVAL;
        case NFPROTO_INET: /* tuple.src.l3num supports NFPROTO_IPV4/6 only */
        default:
                return -EAFNOSUPPORT;
        }

        priv->l4proto = nla_get_u8(tb[NFTA_CT_EXPECT_L4PROTO]);
        switch (priv->l4proto) {
        case IPPROTO_TCP:
        case IPPROTO_UDP:
        case IPPROTO_UDPLITE:
        case IPPROTO_DCCP:
        case IPPROTO_SCTP:
                break;
        default:
                return -EOPNOTSUPP;
        }

        priv->dport = nla_get_be16(tb[NFTA_CT_EXPECT_DPORT]);
        priv->timeout = nla_get_u32(tb[NFTA_CT_EXPECT_TIMEOUT]);
        priv->size = nla_get_u8(tb[NFTA_CT_EXPECT_SIZE]);

        return nf_ct_netns_get(ctx->net, ctx->family);
}

static void nft_ct_expect_obj_destroy(const struct nft_ctx *ctx,
                                       struct nft_object *obj)
{
        nf_ct_netns_put(ctx->net, ctx->family);
}

static int nft_ct_expect_obj_dump(struct sk_buff *skb,
                                  struct nft_object *obj, bool reset)
{
        const struct nft_ct_expect_obj *priv = nft_obj_data(obj);

        if (nla_put_be16(skb, NFTA_CT_EXPECT_L3PROTO, htons(priv->l3num)) ||
            nla_put_u8(skb, NFTA_CT_EXPECT_L4PROTO, priv->l4proto) ||
            nla_put_be16(skb, NFTA_CT_EXPECT_DPORT, priv->dport) ||
            nla_put_u32(skb, NFTA_CT_EXPECT_TIMEOUT, priv->timeout) ||
            nla_put_u8(skb, NFTA_CT_EXPECT_SIZE, priv->size))
                return -1;

        return 0;
}

static void nft_ct_expect_obj_eval(struct nft_object *obj,
                                   struct nft_regs *regs,
                                   const struct nft_pktinfo *pkt)
{
        const struct nft_ct_expect_obj *priv = nft_obj_data(obj);
        struct nf_conntrack_expect *exp;
        enum ip_conntrack_info ctinfo;
        struct nf_conn_help *help;
        enum ip_conntrack_dir dir;
        u16 l3num = priv->l3num;
        struct nf_conn *ct;

        ct = nf_ct_get(pkt->skb, &ctinfo);
        if (!ct || nf_ct_is_confirmed(ct) || nf_ct_is_template(ct)) {
                regs->verdict.code = NFT_BREAK;
                return;
        }
        dir = CTINFO2DIR(ctinfo);

        help = nfct_help(ct);
        if (!help)
                help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
        if (!help) {
                regs->verdict.code = NF_DROP;
                return;
        }

        if (help->expecting[NF_CT_EXPECT_CLASS_DEFAULT] >= priv->size) {
                regs->verdict.code = NFT_BREAK;
                return;
        }
        if (l3num == NFPROTO_INET)
                l3num = nf_ct_l3num(ct);

        exp = nf_ct_expect_alloc(ct);
        if (exp == NULL) {
                regs->verdict.code = NF_DROP;
                return;
        }
        nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, l3num,
                          &ct->tuplehash[!dir].tuple.src.u3,
                          &ct->tuplehash[!dir].tuple.dst.u3,
                          priv->l4proto, NULL, &priv->dport);
        exp->timeout.expires = jiffies + priv->timeout * HZ;

        if (nf_ct_expect_related(exp, 0) != 0)
                regs->verdict.code = NF_DROP;
}

static const struct nla_policy nft_ct_expect_policy[NFTA_CT_EXPECT_MAX + 1] = {
        [NFTA_CT_EXPECT_L3PROTO]        = { .type = NLA_U16 },
        [NFTA_CT_EXPECT_L4PROTO]        = { .type = NLA_U8 },
        [NFTA_CT_EXPECT_DPORT]                = { .type = NLA_U16 },
        [NFTA_CT_EXPECT_TIMEOUT]        = { .type = NLA_U32 },
        [NFTA_CT_EXPECT_SIZE]                = { .type = NLA_U8 },
};

static struct nft_object_type nft_ct_expect_obj_type;

static const struct nft_object_ops nft_ct_expect_obj_ops = {
        .type                = &nft_ct_expect_obj_type,
        .size                = sizeof(struct nft_ct_expect_obj),
        .eval                = nft_ct_expect_obj_eval,
        .init                = nft_ct_expect_obj_init,
        .destroy        = nft_ct_expect_obj_destroy,
        .dump                = nft_ct_expect_obj_dump,
};

static struct nft_object_type nft_ct_expect_obj_type __read_mostly = {
        .type                = NFT_OBJECT_CT_EXPECT,
        .ops                = &nft_ct_expect_obj_ops,
        .maxattr        = NFTA_CT_EXPECT_MAX,
        .policy                = nft_ct_expect_policy,
        .owner                = THIS_MODULE,
};

static int __init nft_ct_module_init(void)
{
        int err;

        BUILD_BUG_ON(NF_CT_LABELS_MAX_SIZE > NFT_REG_SIZE);

        err = nft_register_expr(&nft_ct_type);
        if (err < 0)
                return err;

        err = nft_register_expr(&nft_notrack_type);
        if (err < 0)
                goto err1;

        err = nft_register_obj(&nft_ct_helper_obj_type);
        if (err < 0)
                goto err2;

        err = nft_register_obj(&nft_ct_expect_obj_type);
        if (err < 0)
                goto err3;
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        err = nft_register_obj(&nft_ct_timeout_obj_type);
        if (err < 0)
                goto err4;
#endif
        return 0;

#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
err4:
        nft_unregister_obj(&nft_ct_expect_obj_type);
#endif
err3:
        nft_unregister_obj(&nft_ct_helper_obj_type);
err2:
        nft_unregister_expr(&nft_notrack_type);
err1:
        nft_unregister_expr(&nft_ct_type);
        return err;
}

static void __exit nft_ct_module_exit(void)
{
#ifdef CONFIG_NF_CONNTRACK_TIMEOUT
        nft_unregister_obj(&nft_ct_timeout_obj_type);
#endif
        nft_unregister_obj(&nft_ct_expect_obj_type);
        nft_unregister_obj(&nft_ct_helper_obj_type);
        nft_unregister_expr(&nft_notrack_type);
        nft_unregister_expr(&nft_ct_type);
}

module_init(nft_ct_module_init);
module_exit(nft_ct_module_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Patrick McHardy <kaber@trash.net>");
MODULE_ALIAS_NFT_EXPR("ct");
MODULE_ALIAS_NFT_EXPR("notrack");
MODULE_ALIAS_NFT_OBJ(NFT_OBJECT_CT_HELPER);
MODULE_ALIAS_NFT_OBJ(NFT_OBJECT_CT_TIMEOUT);
MODULE_ALIAS_NFT_OBJ(NFT_OBJECT_CT_EXPECT);
MODULE_DESCRIPTION("Netfilter nf_tables conntrack module");


































    4 



    2 

    3 




    4 



    1 

    3 




   42 




   13 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
// SPDX-License-Identifier: GPL-2.0-only
/*
 *        IEEE 802.1Q Multiple VLAN Registration Protocol (MVRP)
 *
 *        Copyright (c) 2012 Massachusetts Institute of Technology
 *
 *        Adapted from code in net/8021q/vlan_gvrp.c
 *        Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
 */
#include <linux/types.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <net/mrp.h>
#include "vlan.h"

#define MRP_MVRP_ADDRESS        { 0x01, 0x80, 0xc2, 0x00, 0x00, 0x21 }

enum mvrp_attributes {
        MVRP_ATTR_INVALID,
        MVRP_ATTR_VID,
        __MVRP_ATTR_MAX
};
#define MVRP_ATTR_MAX        (__MVRP_A